diff --git a/dir2sync_list b/dir2sync_list new file mode 100644 index 0000000..642392a --- /dev/null +++ b/dir2sync_list @@ -0,0 +1 @@ +/opt/AZURE/BLOB > testcontainer115 diff --git a/notes b/notes new file mode 100644 index 0000000..2a32c67 --- /dev/null +++ b/notes @@ -0,0 +1,11 @@ +cron job using the users cron tab. + +#0 * * * * /usr/bin/env bash -lc '/opt/AZURE/run_azcopy.sh /opt/AZURE/dir2sync_list true 20' + +the baove runs every hour using a bash login shell this is needed so all VARIABLE expansions work correctly. + +We run everything as the user no root. + +there are checks and error hadling was created for some cases but motsliukly not all. So logs should be checked on a regular basis. + +tested on a smal set of files. diff --git a/run_azcopy-multi-dir-multi-container.sh b/run_azcopy-multi-dir-multi-container.sh index c2ff94e..9ec3732 100755 --- a/run_azcopy-multi-dir-multi-container.sh +++ b/run_azcopy-multi-dir-multi-container.sh @@ -4,29 +4,34 @@ cd /opt/AZURE || exit 1 # Configurable variables -BUSINESS_HOURS_START=9 +# Basic variables need for this scrip[t to operate correctly. +BUSINESS_HOURS_START=10 BUSINESS_HOURS_END=20 AZURE_ACCOUNT="" AZURE_SAS="" LOCK_FILE="/tmp/run_azcopy.lock" # Arguments +# From the comand line. the first is mandatory. There is a check for it if its not provided. SOURCE_LIST_FILE="$1" LOGGING="${2:-false}" # Default to false BANDWIDTH_CAP="${3:-0}" # Default is 0 (no cap) # Report files +# A bunch of temp files used to generate logs and a simple completion report. TIMESTAMP=$(date +"%Y%m%d_%H%M%S") LOG_FILE="azcopy_log_$TIMESTAMP.txt" COMPLETION_REPORT="completion_report_$TIMESTAMP.txt" # Ensure source list file is provided +# This the check for source files no ne3ed to run if that is not present. if [[ -z "$SOURCE_LIST_FILE" || ! -f "$SOURCE_LIST_FILE" ]]; then echo "Usage: $0 [log=true|false] [bandwidth_mbps]" exit 1 fi # Lock file to prevent multiple instances +# Part of checks to prevent doubling of running proccesses. if [[ -f "$LOCK_FILE" ]]; then PID=$(cat "$LOCK_FILE") if kill -0 "$PID" 2>/dev/null; then @@ -40,12 +45,15 @@ fi echo $$ > "$LOCK_FILE" # Function to check business hours +# This will ensure we do not run at times when the server might need to do other things. is_business_hours() { - HOUR=$(printf "%d" "$(date +%H)") # Convert to decimal safely + HOUR=$(date +%H | sed 's/^0*//') # Remove leading zeros causing errors at morning times + #HOUR=$(printf "%d" "$(date +%H)") # Convert to decimal safely / this had an issue for some reason not working [[ $HOUR -ge $BUSINESS_HOURS_START && $HOUR -lt $BUSINESS_HOURS_END ]] } # Stop if running during business hours +# Uses the previous funcion to kill the procces if needed. if is_business_hours; then echo "Business hours detected ($BUSINESS_HOURS_START:00 - $BUSINESS_HOURS_END:00). Exiting..." rm -f "$LOCK_FILE" @@ -55,9 +63,13 @@ fi echo "Starting sync job at $(date)" | tee -a "$LOG_FILE" # Read the directory list file into an array +# This is used like that becouse of how cron uses a seperate shell enviroment. mapfile -t SYNC_JOBS < "$SOURCE_LIST_FILE" -# Loop through the array and process each entry +# The actual part of the script that does the job. +# Loops through the array and process each entry +# This also does a check to kill the process if it does not complete before the restriction window apply. +# Also checks if a given container exists will stop if it does not. for LINE in "${SYNC_JOBS[@]}"; do IFS=">" read -r SOURCE_DIR DEST_CONTAINER <<< "$LINE" SOURCE_DIR=$(echo "$SOURCE_DIR" | xargs) # Trim spaces @@ -90,7 +102,7 @@ for LINE in "${SYNC_JOBS[@]}"; do fi rm -f "$TEST_FILE" - # Run azcopy in the foreground (one directory at a time) + # Run azcopy in the background (one directory at a time) if [[ "$LOGGING" == "true" ]]; then azcopy sync "$SOURCE_DIR" "$DEST_URL?$AZURE_SAS" --recursive --cap-mbps "$BANDWIDTH_CAP" | tee -a "$LOG_FILE" & else diff --git a/run_azcopy-rotae-log b/run_azcopy-rotae-log new file mode 100644 index 0000000..c82ab85 --- /dev/null +++ b/run_azcopy-rotae-log @@ -0,0 +1,55 @@ +##Log rotate + +```bash +# /etc/logrotate.d/azure_logs + +/opt/AZURE/*.txt { + weekly + missingok + rotate 4 + compress + delaycompress + notifempty + create 0644 root root +} + +~/.azcopy/*.log { + weekly + missingok + rotate 4 + compress + delaycompress + notifempty + create 0644 root root +} +``` + +### Explanation: +- **weekly**: Rotate the logs on a weekly basis. +- **missingok**: If the log file is missing, go on to the next one without issuing an error message. +- **rotate 4**: Keep only the last 4 weeks of logs. +- **compress**: Compress the rotated logs using gzip. +- **delaycompress**: Delay compression until the next rotation cycle. This means the current log file will not be compressed immediately after rotation, but the previous log files will be. +- **notifempty**: Do not rotate the log if it is empty. +- **create 0644 root root**: Create new log files with owner `root` and group `root`, and permissions `0644`. + +### Steps to Apply the Configuration: +1. Save the above configuration in `/etc/logrotate.d/azure_logs`. +2. Ensure that the `logrotate` service is enabled and running on your system. +3. Test the logrotate configuration with the following command to ensure there are no syntax errors: + +```bash +sudo logrotate -d /etc/logrotate.d/azure_logs +``` + +The `-d` option runs logrotate in debug mode, which will show you what actions would be taken without actually performing them. + +4. If everything looks good, you can force a rotation to test it: + +```bash +sudo logrotate -f /etc/logrotate.d/azure_logs +``` + +This will rotate the logs immediately according to the specified configuration. + +By following these steps, your logs in `/opt/AZURE` and `~/.azcopy` should be rotated weekly, compressed, and kept for only the last 4 weeks. diff --git a/run_azcopy-single-dir.sh b/run_azcopy-single-dir.sh index 3cec300..7925802 100644 --- a/run_azcopy-single-dir.sh +++ b/run_azcopy-single-dir.sh @@ -1,15 +1,18 @@ #!/bin/bash +# Enter the specific 'work directory'Ensure we collect logs and other files in the one place. +cd /opt/AZURE || exit 1 + # Configurable variables -BUSINESS_HOURS_START=7 -BUSINESS_HOURS_END=15 -AZURE_URL="https://<>.core.windows.net/" -AZURE_SAS="" +BUSINESS_HOURS_START=9 +BUSINESS_HOURS_END=20 +AZURE_URL="" +AZURE_SAS="" # Arguments SOURCE_DIR="$1" LOGGING="${2:-false}" # Default to no logs -BANDWIDTH_CAP="${3:-0}" # Default is 0 (no cap) +BANDWIDTH_CAP="${3:-0}" # Default is 0 (no bw limit) # Report files TIMESTAMP=$(date +"%Y%m%d_%H%M%S")