| 0 | 0 |
#!/bin/bash
|
|
1 |
# emacs: -*- mode: python; tab-width: 4; indent-tabs-mode: t -*-
|
|
2 |
# ex: set sts=4 ts=4 sw=4 noet:
|
|
3 |
#
|
| 1 | 4 |
# git-annex-remote-rclone - wrapper to enable use of rclone-supported cloud providers as git-annex special remotes.
|
| 2 | 5 |
#
|
| 3 | 6 |
# Install in PATH as git-annex-remote-rclone
|
| 4 | 7 |
#
|
| 5 | |
# Copyright (C) 2016-2017 Daniel Dent
|
|
8 |
# Copyright (C) 2016-2022 Daniel Dent
|
|
9 |
# 2022 git-annex-remote-rclone contributors
|
| 6 | 10 |
#
|
| 7 | 11 |
# This program is free software: you can redistribute it and/or modify it under the terms of version 3 of the GNU
|
| 8 | 12 |
# General Public License as published by the Free Software Foundation.
|
|
| 43 | 47 |
echo "INITREMOTE-FAILURE rclone_layout setting not recognized"
|
| 44 | 48 |
exit 1
|
| 45 | 49 |
;;
|
| 46 | |
esac
|
|
50 |
esac
|
| 47 | 51 |
}
|
| 48 | 52 |
|
| 49 | 53 |
# Sets LOC to the location to use to store a key.
|
|
| 59 | 63 |
;;
|
| 60 | 64 |
nodir)
|
| 61 | 65 |
LOC="$REMOTE_TARGET:$REMOTE_PREFIX/"
|
| 62 | |
;;
|
|
66 |
;;
|
| 63 | 67 |
mixed)
|
| 64 | 68 |
ask DIRHASH "$1"
|
| 65 | 69 |
LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET"
|
| 66 | 70 |
;;
|
| 67 | 71 |
frankencase)
|
| 68 | 72 |
ask DIRHASH "$1"
|
| 69 | |
lret=$(echo $RET|tr A-Z a-z)
|
|
73 |
lret=$(echo "$RET" | tr '[:upper:]' '[:lower:]')
|
| 70 | 74 |
LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$lret"
|
| 71 | 75 |
;;
|
| 72 | 76 |
esac
|
|
| 76 | 80 |
ask () {
|
| 77 | 81 |
echo "$1" "$2"
|
| 78 | 82 |
read -r resp
|
| 79 | |
# Strip trailing carriage return, if present
|
| 80 | |
resp="${resp%$'\r'}"
|
| 81 | |
if echo $resp|grep '^VALUE '>/dev/null; then
|
| 82 | |
RET=$(echo "$resp" | cut -f2- -d' ')
|
| 83 | |
else
|
| 84 | |
RET=""
|
|
83 |
# Strip trailing carriage return, if present
|
|
84 |
resp="${resp%$'\r'}"
|
|
85 |
if echo "$resp" | grep '^VALUE '>/dev/null; then
|
|
86 |
RET=$(echo "$resp" | cut -f2- -d' ')
|
|
87 |
else
|
|
88 |
RET=""
|
| 85 | 89 |
fi
|
| 86 | 90 |
}
|
| 87 | 91 |
|
|
92 |
GREP () {
|
|
93 |
set +e
|
|
94 |
out=$(grep "$@")
|
|
95 |
rc=$?
|
|
96 |
set -e
|
|
97 |
# Replace explicit newline since we must provide 1 line DEBUG
|
|
98 |
# Fancy sed is from Example 5 of https://linuxhint.com/newline_replace_sed
|
|
99 |
# which worked on Linux and OSX.
|
|
100 |
# shellcheck disable=SC2016
|
|
101 |
out_safe=$(echo "$out" | sed -ne 'H;${x;s/\n/\\n/g;s/^,//;p;}')
|
|
102 |
echo "DEBUG 'grep \"$*\"' exited with rc=$rc and stdout=${out_safe}"
|
|
103 |
exit $rc
|
|
104 |
}
|
| 88 | 105 |
|
| 89 | 106 |
# This has to come first, to get the protocol started.
|
| 90 | 107 |
echo VERSION 1
|
| 91 | 108 |
|
| 92 | 109 |
while read -r line; do
|
| 93 | |
# Strip trailing carriage return, if present
|
| 94 | |
line="${line%$'\r'}"
|
|
110 |
# Strip trailing carriage return, if present
|
|
111 |
line="${line%$'\r'}"
|
|
112 |
# shellcheck disable=SC2086
|
| 95 | 113 |
set -- $line
|
| 96 | 114 |
case "$1" in
|
| 97 | 115 |
INITREMOTE)
|
| 98 | 116 |
# Do anything necessary to create resources
|
| 99 | 117 |
# used by the remote. Try to be idempotent.
|
| 100 | |
#
|
|
118 |
#
|
| 101 | 119 |
# Use GETCONFIG to get any needed configuration
|
| 102 | 120 |
# settings, and SETCONFIG to set any persistent
|
| 103 | 121 |
# configuration settings.
|
| 104 | |
#
|
|
122 |
#
|
| 105 | 123 |
# (Note that this is not run every time, only when
|
| 106 | 124 |
# git annex initremote or git annex enableremote is
|
| 107 | 125 |
# run.)
|
|
| 109 | 127 |
getconfig prefix
|
| 110 | 128 |
REMOTE_PREFIX=$RET
|
| 111 | 129 |
if [ -z "$REMOTE_PREFIX" ]; then
|
| 112 | |
REMOTE_PREFIX="git-annex"
|
|
130 |
REMOTE_PREFIX="git-annex"
|
| 113 | 131 |
fi
|
| 114 | 132 |
if [ "$REMOTE_PREFIX" == "/" ]; then
|
| 115 | |
echo INITREMOTE-FAILURE "storing objects directly in the root (/) is not supported"
|
|
133 |
echo INITREMOTE-FAILURE "storing objects directly in the root (/) is not supported"
|
| 116 | 134 |
fi
|
| 117 | 135 |
setconfig prefix $REMOTE_PREFIX
|
| 118 | 136 |
|
| 119 | 137 |
getconfig target
|
| 120 | 138 |
REMOTE_TARGET=$RET
|
| 121 | |
setconfig target $REMOTE_TARGET
|
| 122 | |
|
|
139 |
setconfig target "$REMOTE_TARGET"
|
|
140 |
|
| 123 | 141 |
getconfig rclone_layout
|
| 124 | 142 |
RCLONE_LAYOUT=$RET
|
| 125 | 143 |
validate_layout
|
| 126 | |
setconfig rclone_layout $RCLONE_LAYOUT
|
|
144 |
setconfig rclone_layout "$RCLONE_LAYOUT"
|
| 127 | 145 |
|
| 128 | 146 |
if [ -z "$REMOTE_TARGET" ]; then
|
| 129 | |
echo INITREMOTE-FAILURE "rclone remote target must be specified (use target= parameter)"
|
| 130 | |
fi
|
| 131 | |
|
| 132 | |
if runcmd rclone mkdir $REMOTE_TARGET:$REMOTE_PREFIX; then
|
| 133 | |
echo INITREMOTE-SUCCESS
|
| 134 | |
else
|
| 135 | |
echo INITREMOTE-FAILURE "Failed to create directory on remote. Ensure that 'rclone config' has been run."
|
| 136 | |
fi
|
|
147 |
echo INITREMOTE-FAILURE "rclone remote target must be specified (use target= parameter)"
|
|
148 |
fi
|
|
149 |
|
|
150 |
if runcmd rclone mkdir "$REMOTE_TARGET:$REMOTE_PREFIX"; then
|
|
151 |
echo INITREMOTE-SUCCESS
|
|
152 |
else
|
|
153 |
echo INITREMOTE-FAILURE "Failed to create directory on remote. Ensure that 'rclone config' has been run."
|
|
154 |
fi
|
| 137 | 155 |
;;
|
| 138 | 156 |
PREPARE)
|
| 139 | 157 |
# Use GETCONFIG to get configuration settings,
|
|
| 142 | 160 |
|
| 143 | 161 |
getconfig prefix
|
| 144 | 162 |
REMOTE_PREFIX="$RET"
|
| 145 | |
|
|
163 |
|
| 146 | 164 |
getconfig target
|
| 147 | 165 |
REMOTE_TARGET="$RET"
|
| 148 | |
|
|
166 |
|
| 149 | 167 |
getconfig rclone_layout
|
| 150 | 168 |
RCLONE_LAYOUT="$RET"
|
| 151 | 169 |
validate_layout
|
| 152 | 170 |
|
| 153 | |
echo PREPARE-SUCCESS
|
|
171 |
echo PREPARE-SUCCESS
|
| 154 | 172 |
;;
|
| 155 | 173 |
TRANSFER)
|
|
174 |
op="$2"
|
| 156 | 175 |
key="$3"
|
| 157 | |
file="$4"
|
| 158 | |
case "$2" in
|
|
176 |
shift 3
|
|
177 |
file="$*"
|
|
178 |
case "$op" in
|
| 159 | 179 |
STORE)
|
| 160 | 180 |
# Store the file to a location
|
| 161 | 181 |
# based on the key.
|
|
| 163 | 183 |
calclocation "$key"
|
| 164 | 184 |
if [ ! -e "$file" ]; then
|
| 165 | 185 |
echo TRANSFER-FAILURE STORE "$key" "asked to store non-existent file $file"
|
| 166 | |
else
|
|
186 |
else
|
| 167 | 187 |
if runcmd rclone copy "$file" "$LOC"; then
|
| 168 | 188 |
echo TRANSFER-SUCCESS STORE "$key"
|
| 169 | 189 |
else
|
| 170 | 190 |
echo TRANSFER-FAILURE STORE "$key"
|
| 171 | 191 |
fi
|
| 172 | |
fi
|
|
192 |
fi
|
| 173 | 193 |
;;
|
| 174 | 194 |
RETRIEVE)
|
| 175 | 195 |
# Retrieve from a location based on
|
|
| 178 | 198 |
calclocation "$key"
|
| 179 | 199 |
# http://stackoverflow.com/questions/31396985/why-is-mktemp-on-os-x-broken-with-a-command-that-worked-on-linux
|
| 180 | 200 |
if GA_RC_TEMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/rclone-annex-tmp.XXXXXXXXX") &&
|
| 181 | |
runcmd rclone copy "$LOC$key" $GA_RC_TEMP_DIR &&
|
| 182 | |
mv $GA_RC_TEMP_DIR/$key $file &&
|
| 183 | |
rmdir $GA_RC_TEMP_DIR; then
|
|
201 |
runcmd rclone copy "$LOC$key" "$GA_RC_TEMP_DIR" &&
|
|
202 |
mv "$GA_RC_TEMP_DIR/$key" "$file" &&
|
|
203 |
rmdir "$GA_RC_TEMP_DIR"; then
|
| 184 | 204 |
echo TRANSFER-SUCCESS RETRIEVE "$key"
|
| 185 | 205 |
else
|
| 186 | 206 |
echo TRANSFER-FAILURE RETRIEVE "$key"
|
|
| 196 | 216 |
# Some rclone backends support multiple
|
| 197 | 217 |
# files under one name.
|
| 198 | 218 |
if check_result=$(rclone size "$LOC$key" 2>&1) &&
|
| 199 | |
echo $check_result|grep -E 'Total objects: [1-9][0-9]* Total size' >&2 &&
|
| 200 | |
! echo $check_result|grep 'Total size: 0 (0 bytes)' >&2; then
|
|
219 |
echo "$check_result" | GREP -E 'Total objects: [1-9][0-9]*\>'; then
|
| 201 | 220 |
echo CHECKPRESENT-SUCCESS "$key"
|
| 202 | 221 |
else
|
| 203 | 222 |
# rclone 1.29 used 'Total objects: 0'
|
| 204 | 223 |
# rclone 1.30 uses 'directory not found'
|
| 205 | |
if echo $check_result|grep 'Total objects: 0' >&2 ||
|
| 206 | |
echo $check_result|grep ' directory not found' >&2; then
|
|
224 |
if echo "$check_result" | GREP 'Total objects: 0' ||
|
|
225 |
echo "$check_result" | GREP ' directory not found'; then
|
| 207 | 226 |
echo CHECKPRESENT-FAILURE "$key"
|
| 208 | 227 |
else
|
| 209 | 228 |
# When the directory does not exist,
|
|
| 221 | 240 |
# Note that it's not a failure to remove a
|
| 222 | 241 |
# key that is not present.
|
| 223 | 242 |
if remove_result=$(rclone delete --retries 1 "$LOC$key" 2>&1); then
|
| 224 | |
echo REMOVE-SUCCESS "$key"
|
| 225 | |
else
|
| 226 | |
# rclone 1.29 used Failed to purge: Couldn't find directory:
|
|
243 |
echo REMOVE-SUCCESS "$key"
|
|
244 |
else
|
|
245 |
# rclone 1.29 used Failed to purge: Couldn't find directory:
|
| 227 | 246 |
# rclone 1.30 used no such file or directory
|
| 228 | 247 |
# rclone 1.33 uses directory not found
|
| 229 | |
if echo $remove_result | grep " Failed to purge: Couldn't find directory: " >&2 ||
|
| 230 | |
echo $remove_result | grep ' no such file or directory' ||
|
| 231 | |
echo $remove_result | grep ' directory not found' >&2
|
| 232 | |
then
|
|
248 |
if echo "$remove_result" | GREP " Failed to purge: Couldn't find directory: " ||
|
|
249 |
echo "$remove_result" | GREP ' no such file or directory' ||
|
|
250 |
echo "$remove_result" | GREP ' directory not found'
|
|
251 |
then
|
| 233 | 252 |
echo REMOVE-SUCCESS "$key"
|
| 234 | |
else
|
| 235 | |
echo REMOVE-FAILURE "$key"
|
|
253 |
else
|
|
254 |
echo REMOVE-FAILURE "$key"
|
| 236 | 255 |
fi
|
| 237 | 256 |
fi
|
| 238 | 257 |
;;
|
|
| 242 | 261 |
# to say that any other request is unsupported.
|
| 243 | 262 |
echo UNSUPPORTED-REQUEST
|
| 244 | 263 |
;;
|
| 245 | |
esac
|
|
264 |
esac
|
| 246 | 265 |
done
|
| 247 | 266 |
|
| 248 | 267 |
|