expoweb/noinfo/rsynccommands.txt

---Notes by Philip Sargent 19/6/2019 updated 10/12/2019
---All these have the -n flag set ("--dry-run") flag set so that idiots just copying 
---things don't do any damage.

---Some systems have problems doing rsync with the ENTIRE expofiles due to deep directories. S
---So do subfolders, e.g. just /photos/ or just /surveyscans/

---This clones expofiles to your PC - if your PC has expofiles in the proper place
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles /home/expo/expofiles

---This clones expofiles to your PC - if you are currently in your expofiles directory
cd expofiles
cd ..
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/ expofiles

---to check progress of the syncronization with the server
cd expofiles
rsync -nazv * expo@expo.survex.com:expofiles/ >rsync-filelist-todo.txt
rsync -nazv rsync*  expo@expo.survex.com:expofiles/

--Finally, to make the server match your local machine
-- the delete option doesn't work for a list of files, the source has to be a directory
cd expofiles
cd ..
rsync -nazv --delete-after --prune-empty-dirs expofiles/ expo@expo.survex.com:expofiles/ 


---
---This clones expofiles to Philip's laptop Barbie only - DO NOT USE unless you are Philip
---
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/             /media/philip/SD-huge/CUCC-Expo/expofiles
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/uploads/     /media/philip/SD-huge/CUCC-Expo/expofiles/uploads
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/writeups/    /media/philip/SD-huge/CUCC-Expo/expofiles/writeups
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/surveyscans/ /media/philip/SD-huge/CUCC-Expo/expofiles/surveyscans
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/photos/      /media/philip/SD-huge/CUCC-Expo/expofiles/photos
rsync -nazv --delete-after --prune-empty-dirs expo@expo.survex.com:expofiles/tunnelwiki/  /media/philip/SD-huge/CUCC-Expo/expofiles/tunnelwiki

--when I copy up to the server I don't want to delete anything (usually)
cd expofiles
rsync -nazv expo@expo.survex.com:expofiles/uploads/ uploads
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/photos/2019/ expo@expo.survex.com:expofiles/photos/2019
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/photos/2018/ expo@expo.survex.com:expofiles/photos/2018
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/gpslogs/     expo@expo.survex.com:expofiles/gpslogs
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/surveyscans/ expo@expo.survex.com:expofiles/surveyscans
rsync -nazv /media/philip/SD-huge/CUCC-Expo/expofiles/             expo@expo.survex.com:expofiles


--or from the Barbie machine you are sitting at to an external drive
rsync -nav --delete-after --prune-empty-dirs /media/philip/SD-huge/CUCC-Expo/expofiles/  /media/philip/150G/expofiles



--- THIS IS ALL HIGHLY SPECIFIC STUFF BELOW when tidying large video files ---

---philip's odd stuff. First copy everything in photos except videos and raw photos
rsync -nazv --exclude=*.PEF --exclude=*.jpg --exclude=*.png --exclude=*.JPG --exclude=*.TIF --exclude=*.AVI --exclude=*.CR2 --exclude=*.ORF --exclude=*.MP4 --exclude=*.avi --exclude=*.jpeg --exclude=*.zip --exclude=*.mp4 --exclude=*.MOV --exclude=*.tif photos/* expo@expo.survex.com:expofiles/photos/

---Unused: What are these things lurking in the photo folders? I have left them in..
 --exclude=*.THM --exclude=*.xls? --exclude=*.txt --exclude=*.TBL --exclude=*.pto --exclude=*.csv --exclude=*.html --exclude=*.php 

---First move just the videos from photos/ to another folder in the source machine
rsync -nazv --remove-source-files --prune-empty-dirs --include=*.AVI --include=*.MP4 --include=*.avi --include=*.mp4 --include=*.MOV --include='*/' --exclude=* photos/*  video2/

---then this to sync to server
rsync -nazv video2/*  expo@expo.survex.com:expofiles/video2/

---Then I have to compress the raw images .PEF .ORF .CR2 
---but it coredumps after each (successful) conversion. So needs to be done in a batch file
---this is a known bug since 2017
ufraw-batch --wb=camera --exposure=auto --out-type=jpeg  --size=1024 *.PEF

for i in $(ls *.PEF); do 
ufraw-batch --wb=camera --exposure=auto --out-type=jpeg  --size=1024 $i; 
done

---but ufraw-batch can't open the TIF files, so I have to use convert
convert -resize "1024>" *.TIF  *.jpg
---but I can't get intelligent file renaming, so again I use a script (which creates bogus thumbs which I delete by hand)
for i in $(ls *.TIF); do 
convert $i -resize "1024>" $i.jpg 
done

---videos, first without the big derig & strublamm
rsync -nazv --prune-empty-dirs --exclude='derigvideo*' --exclude='strubklamm*' --include=*.AVI --include=*.MP4 --include=*.avi --include=*.mp4 --include=*.MOV --include='*/' --exclude=* video/*  expo@expo.survex.com:expofiles/video/

---tidy up scripts and output
rsync -nazv rsync*  expo@expo.survex.com:expofiles/

---more temporary fix stuff
cd expofiles
rsync -nazv photos/2019/unsorted/surveyscans-2019/ surveyscans/2019
rsync -nazv --delete-after surveyscans/2019/ photos/2019/unsorted/surveyscans-2019