small fix / now tested working

dir2sync_list

@@ -0,0 +1 @@
+/opt/AZURE/BLOB > testcontainer115

notes

@@ -0,0 +1,11 @@
+cron job using the users cron tab. 
+
+#0 * * * * /usr/bin/env bash -lc '/opt/AZURE/run_azcopy.sh /opt/AZURE/dir2sync_list true 20'
+
+the baove runs every hour using a bash login shell this is needed so all VARIABLE expansions work correctly. 
+
+We run everything as the user no root.
+
+there are checks and error hadling was created for some cases but motsliukly not all. So logs should be checked on a regular basis. 
+
+tested on a smal set of files.

run_azcopy-multi-dir-multi-container.sh

@@ -4,29 +4,34 @@
 cd /opt/AZURE || exit 1
 
 # Configurable variables
-BUSINESS_HOURS_START=9
+# Basic variables need for this scrip[t to operate correctly. 
+BUSINESS_HOURS_START=10
 BUSINESS_HOURS_END=20
 AZURE_ACCOUNT=""
 AZURE_SAS=""
 LOCK_FILE="/tmp/run_azcopy.lock"
 
 # Arguments
+# From the comand line. the first is mandatory. There is a check for it if its not provided.
 SOURCE_LIST_FILE="$1"
 LOGGING="${2:-false}"  # Default to false
 BANDWIDTH_CAP="${3:-0}"  # Default is 0 (no cap)
 
 # Report files
+# A bunch of temp files used to generate logs and a simple completion report.
 TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
 LOG_FILE="azcopy_log_$TIMESTAMP.txt"
 COMPLETION_REPORT="completion_report_$TIMESTAMP.txt"
 
 # Ensure source list file is provided
+# This the check for source files no ne3ed to run if that is not present. 
 if [[ -z "$SOURCE_LIST_FILE" || ! -f "$SOURCE_LIST_FILE" ]]; then
     echo "Usage: $0 <directory_list_file> [log=true|false] [bandwidth_mbps]"
     exit 1
 fi
 
 # Lock file to prevent multiple instances
+# Part of checks to prevent doubling of running proccesses.
 if [[ -f "$LOCK_FILE" ]]; then
     PID=$(cat "$LOCK_FILE")
     if kill -0 "$PID" 2>/dev/null; then
@@ -40,12 +45,15 @@ fi
 echo $$ > "$LOCK_FILE"
 
 # Function to check business hours
+# This will ensure we do not run at times when the server might need to do other things.
 is_business_hours() {
-    HOUR=$(printf "%d" "$(date +%H)")  # Convert to decimal safely
+    HOUR=$(date +%H | sed 's/^0*//') # Remove leading zeros causing errors at morning times
+    #HOUR=$(printf "%d" "$(date +%H)")  # Convert to decimal safely / this had an issue for some reason not working
     [[ $HOUR -ge $BUSINESS_HOURS_START && $HOUR -lt $BUSINESS_HOURS_END ]]
 }
 
 # Stop if running during business hours
+# Uses the previous funcion to kill the procces if needed.
 if is_business_hours; then
     echo "Business hours detected ($BUSINESS_HOURS_START:00 - $BUSINESS_HOURS_END:00). Exiting..."
     rm -f "$LOCK_FILE"
@@ -55,9 +63,13 @@ fi
 echo "Starting sync job at $(date)" | tee -a "$LOG_FILE"
 
 # Read the directory list file into an array
+# This is used like that becouse of how cron uses a seperate shell enviroment.
 mapfile -t SYNC_JOBS < "$SOURCE_LIST_FILE"
 
-# Loop through the array and process each entry
+# The actual part of the script that does the job.
+# Loops through the array and process each entry
+# This also does a check to kill the process if it does not complete before the restriction window apply.
+# Also checks if a given container exists will stop if it does not.
 for LINE in "${SYNC_JOBS[@]}"; do
     IFS=">" read -r SOURCE_DIR DEST_CONTAINER <<< "$LINE"
     SOURCE_DIR=$(echo "$SOURCE_DIR" | xargs)   # Trim spaces
@@ -90,7 +102,7 @@ for LINE in "${SYNC_JOBS[@]}"; do
     fi
     rm -f "$TEST_FILE"
 
-    # Run azcopy in the foreground (one directory at a time)
+    # Run azcopy in the background (one directory at a time)
     if [[ "$LOGGING" == "true" ]]; then
         azcopy sync "$SOURCE_DIR" "$DEST_URL?$AZURE_SAS" --recursive --cap-mbps "$BANDWIDTH_CAP" | tee -a "$LOG_FILE" &
     else

run_azcopy-rotae-log

@@ -0,0 +1,55 @@
+##Log rotate
+
+```bash
+# /etc/logrotate.d/azure_logs
+
+/opt/AZURE/*.txt {
+    weekly
+    missingok
+    rotate 4
+    compress
+    delaycompress
+    notifempty
+    create 0644 root root
+}
+
+~/.azcopy/*.log {
+    weekly
+    missingok
+    rotate 4
+    compress
+    delaycompress
+    notifempty
+    create 0644 root root
+}
+```
+
+### Explanation:
+- **weekly**: Rotate the logs on a weekly basis.
+- **missingok**: If the log file is missing, go on to the next one without issuing an error message.
+- **rotate 4**: Keep only the last 4 weeks of logs.
+- **compress**: Compress the rotated logs using gzip.
+- **delaycompress**: Delay compression until the next rotation cycle. This means the current log file will not be compressed immediately after rotation, but the previous log files will be.
+- **notifempty**: Do not rotate the log if it is empty.
+- **create 0644 root root**: Create new log files with owner `root` and group `root`, and permissions `0644`.
+
+### Steps to Apply the Configuration:
+1. Save the above configuration in `/etc/logrotate.d/azure_logs`.
+2. Ensure that the `logrotate` service is enabled and running on your system.
+3. Test the logrotate configuration with the following command to ensure there are no syntax errors:
+
+```bash
+sudo logrotate -d /etc/logrotate.d/azure_logs
+```
+
+The `-d` option runs logrotate in debug mode, which will show you what actions would be taken without actually performing them.
+
+4. If everything looks good, you can force a rotation to test it:
+
+```bash
+sudo logrotate -f /etc/logrotate.d/azure_logs
+```
+
+This will rotate the logs immediately according to the specified configuration.
+
+By following these steps, your logs in `/opt/AZURE` and `~/.azcopy` should be rotated weekly, compressed, and kept for only the last 4 weeks.

run_azcopy-single-dir.sh

@@ -1,15 +1,18 @@
 #!/bin/bash
 
+# Enter the specific 'work directory'Ensure we collect logs and other files in the one place.
+cd /opt/AZURE || exit 1
+
 # Configurable variables
-BUSINESS_HOURS_START=7
-BUSINESS_HOURS_END=15
-AZURE_URL="https://<>.core.windows.net/<container>"
-AZURE_SAS="<add key here>"
+BUSINESS_HOURS_START=9
+BUSINESS_HOURS_END=20
+AZURE_URL=""
+AZURE_SAS=""
 
 # Arguments
 SOURCE_DIR="$1"
 LOGGING="${2:-false}" # Default to no logs
-BANDWIDTH_CAP="${3:-0}"  # Default is 0 (no cap)
+BANDWIDTH_CAP="${3:-0}"  # Default is 0 (no bw limit)
 
 # Report files
 TIMESTAMP=$(date +"%Y%m%d_%H%M%S")