expo/expoweb
1
0
Fork 0
mirror of https://expo.survex.com/repositories/expoweb/.git/ synced 2024-11-24 16:21:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fiddling with rsync problems

Browse Source
This commit is contained in:
Philip Sargent 2019-12-16 15:01:51 +00:00
parent a95843d55b
commit 877322cf46
2 changed files with 3509 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3413
noinfo/rsync-problems.txt Normal file
View File

File diff suppressed because it is too large Load Diff

96
noinfo/rsynccommands.txt Normal file
View File

@ -0,0 +1,96 @@
---Notes by Philip Sargent 19/6/2019 updated 10/12/2019
---All these have the -n flag set ("--dry-run") flag set so that idiots just copying
---things don't do any damage.
---Some systems have problems doing rsync with the ENTIRE expofiles due to deep directories. S
---So do subfolders, e.g. just /photos/ or just /surveyscans/
---This clones expofiles to your PC - if your PC has expofiles in the proper place
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles /home/expo/expofiles
---This clones expofiles to your PC - if you are currently in your expofiles directory
cd expofiles
cd ..
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/ expofiles
---to check progress of the syncronization with the server
cd expofiles
rsync -nazv * expo@expo.survex.com:expofiles/ >rsync-filelist-todo.txt
rsync -nazv rsync* expo@expo.survex.com:expofiles/
--Finally, to make the server match your local machine
-- the delete option doesn't work for a list of files, the source has to be a directory
cd expofiles
cd ..
rsync -nazv --delete-after --prune-empty-dirs expofiles/ expo@expo.survex.com:expofiles/
---
---This clones expofiles to Philip's laptop Barbie only - DO NOT USE unless you are Philip
---
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/ /media/philip/SD-huge/CUCC-Expo/expofiles
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/uploads/ /media/philip/SD-huge/CUCC-Expo/expofiles/uploads
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/writeups/ /media/philip/SD-huge/CUCC-Expo/expofiles/writeups
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/surveyscans/ /media/philip/SD-huge/CUCC-Expo/expofiles/surveyscans
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/photos/ /media/philip/SD-huge/CUCC-Expo/expofiles/photos
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/tunnelwiki/ /media/philip/SD-huge/CUCC-Expo/expofiles/tunnelwiki
--when I copy up to the server I don't want to delete anything (usually)
cd expofiles
rsync -nazv expo@expo.survex.com:expofiles/uploads/ uploads
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/photos/2019/ expo@expo.survex.com:expofiles/photos/2019
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/photos/2018/ expo@expo.survex.com:expofiles/photos/2018
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/gpslogs/ expo@expo.survex.com:expofiles/gpslogs
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/surveyscans/ expo@expo.survex.com:expofiles/surveyscans
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/ expo@expo.survex.com:expofiles
--or from the Barbie machine you are sitting at to an external drive
rsync -nav --delete-after --prune-empty-dirs /media/philip/SD-huge/CUCC-Expo/expofiles/ /media/philip/150G/expofiles
--- THIS IS ALL HIGHLY SPECIFIC STUFF BELOW when tidying large video files ---
---philip's odd stuff. First copy everything in photos except videos and raw photos
rsync -nazv --exclude=*.PEF --exclude=*.jpg --exclude=*.png --exclude=*.JPG --exclude=*.TIF --exclude=*.AVI --exclude=*.CR2 --exclude=*.ORF --exclude=*.MP4 --exclude=*.avi --exclude=*.jpeg --exclude=*.zip --exclude=*.mp4 --exclude=*.MOV --exclude=*.tif photos/* expo@expo.survex.com:expofiles/photos/
---Unused: What are these things lurking in the photo folders? I have left them in..
--exclude=*.THM --exclude=*.xls? --exclude=*.txt --exclude=*.TBL --exclude=*.pto --exclude=*.csv --exclude=*.html --exclude=*.php
---First move just the videos from photos/ to another folder in the source machine
rsync -nazv --remove-source-files --prune-empty-dirs --include=*.AVI --include=*.MP4 --include=*.avi --include=*.mp4 --include=*.MOV --include='*/' --exclude=* photos/* video2/
---then this to sync to server
rsync -nazv video2/* expo@expo.survex.com:expofiles/video2/
---Then I have to compress the raw images .PEF .ORF .CR2
---but it coredumps after each (successful) conversion. So needs to be done in a batch file
---this is a known bug since 2017
ufraw-batch --wb=camera --exposure=auto --out-type=jpeg --size=1024 *.PEF
for i in $(ls *.PEF); do
ufraw-batch --wb=camera --exposure=auto --out-type=jpeg --size=1024 $i;
done
---but ufraw-batch can't open the TIF files, so I have to use convert
convert -resize "1024>" *.TIF *.jpg
---but I can't get intelligent file renaming, so again I use a script (which creates bogus thumbs which I delete by hand)
for i in $(ls *.TIF); do
convert $i -resize "1024>" $i.jpg
done
---videos, first without the big derig & strublamm
rsync -nazv --prune-empty-dirs --exclude='derigvideo*' --exclude='strubklamm*' --include=*.AVI --include=*.MP4 --include=*.avi --include=*.mp4 --include=*.MOV --include='*/' --exclude=* video/* expo@expo.survex.com:expofiles/video/
---tidy up scripts and output
rsync -nazv rsync* expo@expo.survex.com:expofiles/
---more temporary fix stuff
cd expofiles
rsync -nazv photos/2019/unsorted/surveyscans-2019/ surveyscans/2019
rsync -nazv --delete-after surveyscans/2019/ photos/2019/unsorted/surveyscans-2019