multi dir / multi container version

run_azcopy-multi-dir-multi-container.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Enter the specific 'work directory'Ensure we collect logs and other files in the one place.
+cd /opt/AZURE || exit 1
+
+# Configurable variables
+BUSINESS_HOURS_START=9
+BUSINESS_HOURS_END=17
+AZURE_ACCOUNT=""
+AZURE_SAS=""
+LOCK_FILE="/tmp/run_azcopy.lock"
+
+# Arguments
+SOURCE_LIST_FILE="$1"
+LOGGING="${2:-false}"  # Default to false
+BANDWIDTH_CAP="${3:-0}"  # Default is 0 (no cap)
+
+# Report files
+TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+LOG_FILE="azcopy_log_$TIMESTAMP.txt"
+COMPLETION_REPORT="completion_report_$TIMESTAMP.txt"
+
+# Ensure source list file is provided
+if [[ -z "$SOURCE_LIST_FILE" || ! -f "$SOURCE_LIST_FILE" ]]; then
+    echo "Usage: $0 <directory_list_file> [log=true|false] [bandwidth_mbps]"
+    exit 1
+fi
+
+# Lock file to prevent multiple instances
+if [[ -f "$LOCK_FILE" ]]; then
+    PID=$(cat "$LOCK_FILE")
+    if kill -0 "$PID" 2>/dev/null; then
+        echo "Another instance (PID $PID) is already running. Exiting..."
+        exit 1
+    else
+        echo "Stale lock file found. Removing..."
+        rm -f "$LOCK_FILE"
+    fi
+fi
+echo $$ > "$LOCK_FILE"
+
+# Function to check business hours
+is_business_hours() {
+    HOUR=$(printf "%d" "$(date +%H)")  # Convert to decimal safely
+    [[ $HOUR -ge $BUSINESS_HOURS_START && $HOUR -lt $BUSINESS_HOURS_END ]]
+}
+
+# Stop if running during business hours
+if is_business_hours; then
+    echo "Business hours detected ($BUSINESS_HOURS_START:00 - $BUSINESS_HOURS_END:00). Exiting..."
+    rm -f "$LOCK_FILE"
+    exit 1
+fi
+
+echo "Starting sync job at $(date)" | tee -a "$LOG_FILE"
+
+# Loop through directories in the list
+while IFS=">" read -r SOURCE_DIR DEST_CONTAINER; do
+    SOURCE_DIR=$(echo "$SOURCE_DIR" | xargs)   # Trim spaces
+    DEST_CONTAINER=$(echo "$DEST_CONTAINER" | xargs)  # Trim spaces
+
+    if [[ -z "$SOURCE_DIR" || ! -d "$SOURCE_DIR" ]]; then
+        echo "ERROR: Invalid directory: $SOURCE_DIR. Exiting." | tee -a "$LOG_FILE"
+        rm -f "$LOCK_FILE"
+        exit 1
+    fi
+
+    if [[ -z "$DEST_CONTAINER" ]]; then
+        echo "ERROR: No destination container specified for $SOURCE_DIR. Exiting." | tee -a "$LOG_FILE"
+        rm -f "$LOCK_FILE"
+        exit 1
+    fi
+
+    DEST_URL="$AZURE_ACCOUNT/$DEST_CONTAINER"
+
+    echo "Syncing $SOURCE_DIR to container: $DEST_CONTAINER" | tee -a "$LOG_FILE"
+
+    # Check if the container exists by attempting to write a small test file
+    TEST_FILE="$SOURCE_DIR/.azcopy_test_file"
+    touch "$TEST_FILE"
+    azcopy cp "$TEST_FILE" "$DEST_URL?$AZURE_SAS" > /dev/null 2>&1
+    if [[ $? -ne 0 ]]; then
+        echo "ERROR: Destination container $DEST_CONTAINER does not exist or is inaccessible. Exiting." | tee -a "$LOG_FILE"
+        rm -f "$TEST_FILE"
+        rm -f "$LOCK_FILE"
+        exit 1
+    fi
+    rm -f "$TEST_FILE"
+
+    # Run azcopy for actual sync in the background
+    if [[ "$LOGGING" == "true" ]]; then
+        azcopy sync "$SOURCE_DIR" "$DEST_URL?$AZURE_SAS" --recursive --cap-mbps "$BANDWIDTH_CAP" | tee -a "$LOG_FILE" &
+    else
+        azcopy sync "$SOURCE_DIR" "$DEST_URL?$AZURE_SAS" --recursive --cap-mbps "$BANDWIDTH_CAP" > /dev/null 2>&1 &
+    fi
+    AZCOPY_PID=$!
+
+    # Monitor the process
+    while kill -0 $AZCOPY_PID 2>/dev/null; do
+        if is_business_hours; then
+            echo -e "\nBusiness hours started! Stopping azcopy..." | tee -a "$LOG_FILE"
+            kill $AZCOPY_PID
+            wait $AZCOPY_PID 2>/dev/null  # Ensure the process is fully stopped
+            rm -f "$LOCK_FILE"
+            exit 1
+        fi
+        sleep 30  # Check every 30 seconds
+    done
+
+    # Check for sync failure
+    if [[ $? -ne 0 ]]; then
+        echo "ERROR: Sync failed for $SOURCE_DIR to $DEST_CONTAINER. Stopping script." | tee -a "$LOG_FILE"
+        rm -f "$LOCK_FILE"
+        exit 1
+    fi
+
+done < "$SOURCE_LIST_FILE"
+
+echo "All directories synced successfully!" | tee -a "$LOG_FILE"
+
+# Generate completion report
+echo "Sync Completed: $(date)" > "$COMPLETION_REPORT"
+echo "All directories listed in $SOURCE_LIST_FILE have been synced." >> "$COMPLETION_REPORT"
+
+echo "Completion report generated: $COMPLETION_REPORT"
+rm -f "$LOCK_FILE"
+exit 0
+