Cloudplow
Automatic rclone remote uploader, with support for multiple remote/folder pairings. UnionFS Cleaner functionality: Deletion of UnionFS whiteout files and their corresponding files on rclone remotes. Automatic remote syncer: Sync between different remotes via a Scaleway server instance, that is created and destroyed at every sync.
Install / Use
/learn @l3uddz/CloudplowREADME
<!-- TOC depthFrom:1 depthTo:2 withLinks:1 updateOnSave:1 orderedList:0 --> <!-- /TOC -->
Introduction
Cloudplow has 3 main functions:
-
Automatic uploader to Rclone remote : Files are moved off local storage. With support for multiple uploaders (i.e. remote/folder pairings).
-
UnionFS Cleaner functionality: Deletion of UnionFS-Fuse whiteout files (
*_HIDDEN~) and their corresponding "whited-out" files on Rclone remotes. With support for multiple remotes (useful if you have multiple Rclone remotes mounted). -
Automatic remote syncer: Sync between two different Rclone remotes using 3rd party VM instances. With support for multiple remote/folder pairings. With support for multiple syncers (i.e. remote/remote pairings).
Requirements
-
Ubuntu/Debian OS (could work in other OSes with some tweaks).
-
Python 3.6 or higher (
sudo apt install python3 python3-pip). -
Required Python modules (see below).
Installation
-
Clone the Cloudplow repo.
sudo git clone https://github.com/l3uddz/cloudplow /opt/cloudplow -
Fix permissions of the Cloudplow folder (replace
user/groupwith your info; runidto check).sudo chown -R user:group /opt/cloudplow -
Go into the Cloudplow folder.
cd /opt/cloudplow -
Install Python PIP.
sudo apt-get install python3-pip -
Install the required python modules.
sudo python3 -m pip install -r requirements.txt -
Create a shortcut for Cloudplow.
sudo ln -s /opt/cloudplow/cloudplow.py /usr/local/bin/cloudplow -
Generate a basic
config.jsonfile.cloudplow run -
Configure the
config.jsonfile.nano config.json
Configuration
Sample
{
"core": {
"dry_run": false,
"rclone_binary_path": "/usr/bin/rclone",
"rclone_config_path": "/home/seed/.config/rclone/rclone.conf"
},
"hidden": {
"/mnt/local/.unionfs-fuse": {
"hidden_remotes": [
"google"
]
}
},
"notifications": {
"Pushover": {
"app_token": "",
"service": "pushover",
"user_token": "",
"priority": "0"
},
"Slack": {
"webhook_url": "",
"sender_name": "cloudplow",
"sender_icon": ":heavy_exclamation_mark:",
"channel": "",
"service": "slack"
}
},
"nzbget": {
"enabled": false,
"url": "https://user:pass@nzbget.domain.com"
},
"sabnzbd": {
"enabled": false,
"url": "https://sabnzbd.domain.com",
"api-key": "1314234234"
},
"plex": {
"enabled": true,
"max_streams_before_throttle": 1,
"ignore_local_streams": true,
"poll_interval": 60,
"notifications": false,
"rclone": {
"throttle_speeds": {
"0": "100M",
"1": "50M",
"2": "40M",
"3": "30M",
"4": "20M",
"5": "10M"
},
"url": "http://localhost:7949"
},
"token": "",
"url": "https://plex.domain.com"
},
"remotes": {
"google": {
"hidden_remote": "google:",
"rclone_excludes": [
"**partial~",
"**_HIDDEN~",
".unionfs/**",
".unionfs-fuse/**"
],
"rclone_extras": {
"--checkers": 16,
"--drive-chunk-size": "64M",
"--stats": "60s",
"--transfers": 8,
"--verbose": 1,
"--skip-links": null,
"--drive-stop-on-upload-limit": null,
"--user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"
},
"rclone_sleeps": {
"Failed to copy: googleapi: Error 403: User rate limit exceeded": {
"count": 5,
"sleep": 25,
"timeout": 3600
},
" 0/s,": {
"count": 15,
"sleep": 25,
"timeout": 140
}
},
"rclone_command": "move",
"remove_empty_dir_depth": 2,
"sync_remote": "google:/Backups",
"upload_folder": "/mnt/local/Media",
"upload_remote": "google:/Media"
},
"google_downloads": {
"hidden_remote": "",
"rclone_excludes": [
"**partial~",
"**_HIDDEN~",
".unionfs/**",
".unionfs-fuse/**"
],
"rclone_extras": {
"--checkers": 32,
"--stats": "60s",
"--transfers": 16,
"--verbose": 1,
"--skip-links": null
},
"rclone_sleeps": {
},
"rclone_command": "copy",
"remove_empty_dir_depth": 2,
"sync_remote": "",
"upload_folder": "/mnt/local/Downloads",
"upload_remote": "google:/Downloads"
},
"box": {
"hidden_remote": "box:",
"rclone_excludes": [
"**partial~",
"**_HIDDEN~",
".unionfs/**",
".unionfs-fuse/**"
],
"rclone_extras": {
"--user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
"--checkers": 32,
"--stats": "60s",
"--transfers": 16,
"--verbose": 1,
"--skip-links": null
},
"rclone_sleeps": {
"Failed to copy: googleapi: Error 403: User rate limit exceeded": {
"count": 5,
"sleep": 25,
"timeout": 300
},
" 0/s,": {
"count": 15,
"sleep": 25,
"timeout": 140
}
},
"rclone_command": "move",
"remove_empty_dir_depth": 2,
"sync_remote": "box:/Backups",
"upload_folder": "/mnt/local/Media",
"upload_remote": "box:/Media"
},
"google_with_mover": {
"hidden_remote": "google:",
"rclone_excludes": [
"**partial~",
"**_HIDDEN~",
".unionfs/**",
".unionfs-fuse/**"
],
"rclone_extras": {
"--checkers": 16,
"--drive-chunk-size": "64M",
"--stats": "60s",
"--transfers": 8,
"--verbose": 1,
"--skip-links": null,
"--drive-stop-on-upload-limit": null, <-- this one
"--user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"
},
"rclone_sleeps": {
"Failed to copy: googleapi: Error 403: User rate limit exceeded": {
"count": 5,
"sleep": 25,
"timeout": 3600
},
" 0/s,": {
"count": 15,
"sleep": 25,
"timeout": 140
}
},
"rclone_command": "move",
"remove_empty_dir_depth": 2,
"sync_remote": "google:/Backups",
"upload_folder": "/mnt/local/Media",
"upload_remote": "google:/Media"
}
},
"syncer": {
"google2box": {
"rclone_extras": {
"--bwlimit": "80M",
"--checkers": 32,
"--drive-chunk-size": "64M",
"--stats": "60s",
"--transfers": 16,
"--verbose": 1
},
"service": "scaleway",
"sync_from": "google",
"sync_interval": 24,
"sync_to": "box",
"tool_path": "/home/seed/go/bin/scw",
"use_copy": true,
"instance_destroy": false
}
},
"uploader": {
"google": {
"can_be_throttled": true,
"check_interval": 30,
"exclude_open_f
