Compare commits
2 Commits
8e370e50d3
...
e7c1154372
| Author | SHA1 | Date | |
|---|---|---|---|
|
e7c1154372 | ||
|
|
ffd8fb4a34 |
128
run_azcopy-multi-dir-multi-container.sh
Normal file
128
run_azcopy-multi-dir-multi-container.sh
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Enter the specific 'work directory'Ensure we collect logs and other files in the one place.
|
||||
cd /opt/AZURE || exit 1
|
||||
|
||||
# Configurable variables
|
||||
BUSINESS_HOURS_START=9
|
||||
BUSINESS_HOURS_END=17
|
||||
AZURE_ACCOUNT=""
|
||||
AZURE_SAS=""
|
||||
LOCK_FILE="/tmp/run_azcopy.lock"
|
||||
|
||||
# Arguments
|
||||
SOURCE_LIST_FILE="$1"
|
||||
LOGGING="${2:-false}" # Default to false
|
||||
BANDWIDTH_CAP="${3:-0}" # Default is 0 (no cap)
|
||||
|
||||
# Report files
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
LOG_FILE="azcopy_log_$TIMESTAMP.txt"
|
||||
COMPLETION_REPORT="completion_report_$TIMESTAMP.txt"
|
||||
|
||||
# Ensure source list file is provided
|
||||
if [[ -z "$SOURCE_LIST_FILE" || ! -f "$SOURCE_LIST_FILE" ]]; then
|
||||
echo "Usage: $0 <directory_list_file> [log=true|false] [bandwidth_mbps]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Lock file to prevent multiple instances
|
||||
if [[ -f "$LOCK_FILE" ]]; then
|
||||
PID=$(cat "$LOCK_FILE")
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
echo "Another instance (PID $PID) is already running. Exiting..."
|
||||
exit 1
|
||||
else
|
||||
echo "Stale lock file found. Removing..."
|
||||
rm -f "$LOCK_FILE"
|
||||
fi
|
||||
fi
|
||||
echo $$ > "$LOCK_FILE"
|
||||
|
||||
# Function to check business hours
|
||||
is_business_hours() {
|
||||
HOUR=$(printf "%d" "$(date +%H)") # Convert to decimal safely
|
||||
[[ $HOUR -ge $BUSINESS_HOURS_START && $HOUR -lt $BUSINESS_HOURS_END ]]
|
||||
}
|
||||
|
||||
# Stop if running during business hours
|
||||
if is_business_hours; then
|
||||
echo "Business hours detected ($BUSINESS_HOURS_START:00 - $BUSINESS_HOURS_END:00). Exiting..."
|
||||
rm -f "$LOCK_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting sync job at $(date)" | tee -a "$LOG_FILE"
|
||||
|
||||
# Loop through directories in the list
|
||||
while IFS=">" read -r SOURCE_DIR DEST_CONTAINER; do
|
||||
SOURCE_DIR=$(echo "$SOURCE_DIR" | xargs) # Trim spaces
|
||||
DEST_CONTAINER=$(echo "$DEST_CONTAINER" | xargs) # Trim spaces
|
||||
|
||||
if [[ -z "$SOURCE_DIR" || ! -d "$SOURCE_DIR" ]]; then
|
||||
echo "ERROR: Invalid directory: $SOURCE_DIR. Exiting." | tee -a "$LOG_FILE"
|
||||
rm -f "$LOCK_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$DEST_CONTAINER" ]]; then
|
||||
echo "ERROR: No destination container specified for $SOURCE_DIR. Exiting." | tee -a "$LOG_FILE"
|
||||
rm -f "$LOCK_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DEST_URL="$AZURE_ACCOUNT/$DEST_CONTAINER"
|
||||
|
||||
echo "Syncing $SOURCE_DIR to container: $DEST_CONTAINER" | tee -a "$LOG_FILE"
|
||||
|
||||
# Check if the container exists by attempting to write a small test file
|
||||
TEST_FILE="$SOURCE_DIR/.azcopy_test_file"
|
||||
touch "$TEST_FILE"
|
||||
azcopy cp "$TEST_FILE" "$DEST_URL?$AZURE_SAS" > /dev/null 2>&1
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "ERROR: Destination container $DEST_CONTAINER does not exist or is inaccessible. Exiting." | tee -a "$LOG_FILE"
|
||||
rm -f "$TEST_FILE"
|
||||
rm -f "$LOCK_FILE"
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$TEST_FILE"
|
||||
|
||||
# Run azcopy for actual sync in the background
|
||||
if [[ "$LOGGING" == "true" ]]; then
|
||||
azcopy sync "$SOURCE_DIR" "$DEST_URL?$AZURE_SAS" --recursive --cap-mbps "$BANDWIDTH_CAP" | tee -a "$LOG_FILE" &
|
||||
else
|
||||
azcopy sync "$SOURCE_DIR" "$DEST_URL?$AZURE_SAS" --recursive --cap-mbps "$BANDWIDTH_CAP" > /dev/null 2>&1 &
|
||||
fi
|
||||
AZCOPY_PID=$!
|
||||
|
||||
# Monitor the process
|
||||
while kill -0 $AZCOPY_PID 2>/dev/null; do
|
||||
if is_business_hours; then
|
||||
echo -e "\nBusiness hours started! Stopping azcopy..." | tee -a "$LOG_FILE"
|
||||
kill $AZCOPY_PID
|
||||
wait $AZCOPY_PID 2>/dev/null # Ensure the process is fully stopped
|
||||
rm -f "$LOCK_FILE"
|
||||
exit 1
|
||||
fi
|
||||
sleep 30 # Check every 30 seconds
|
||||
done
|
||||
|
||||
# Check for sync failure
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "ERROR: Sync failed for $SOURCE_DIR to $DEST_CONTAINER. Stopping script." | tee -a "$LOG_FILE"
|
||||
rm -f "$LOCK_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
done < "$SOURCE_LIST_FILE"
|
||||
|
||||
echo "All directories synced successfully!" | tee -a "$LOG_FILE"
|
||||
|
||||
# Generate completion report
|
||||
echo "Sync Completed: $(date)" > "$COMPLETION_REPORT"
|
||||
echo "All directories listed in $SOURCE_LIST_FILE have been synced." >> "$COMPLETION_REPORT"
|
||||
|
||||
echo "Completion report generated: $COMPLETION_REPORT"
|
||||
rm -f "$LOCK_FILE"
|
||||
exit 0
|
||||
|
||||
101
tests.md
101
tests.md
@@ -1,101 +0,0 @@
|
||||
#1st test.
|
||||
|
||||
Just run.
|
||||
|
||||
Detected busines hours and stopped.
|
||||
|
||||
---
|
||||
[test@alma-azure-test AZURE]$ ./run_azcopy.sh /opt/AZURE/ true 20
|
||||
Business hours detected (9:00 - 17:00). Exiting...
|
||||
|
||||
#2nd test.
|
||||
|
||||
Changed busines hours to something outside the time of test. To se if it operates as expected. 13 to 16
|
||||
|
||||
Started sync just before 13 to se if a file gets copied to blob and if the bandwith limit works and that the script gets stopped before finishing becouse going over after 13
|
||||
|
||||
---
|
||||
[test@alma-azure-test AZURE]$ ./run_azcopy.sh /opt/AZURE/ true 10 &
|
||||
[2] 8109
|
||||
[test@alma-azure-test AZURE]$ Initial file list created: 8 files found.
|
||||
Running: azcopy sync "/opt/AZURE/" "https://115.blob.core.windows.net/115?sv=2022-11-02&ss=bfqt&srt=sco&sp=rwlacupitfx&se=2025-02-20T18:39:49Z&st=2025-02-19T10:39:49Z&spr=https&sig=xxxx" --recursive --cap-mbps 10 &
|
||||
azcopy started with PID 8128
|
||||
Error: 2 arguments source and destination are required for this command. Number of commands passed 3
|
||||
|
||||
Issues going to background.
|
||||
|
||||
#3rd test.
|
||||
|
||||
Amended the way fucnion calls azcopy to beter handle backgrouind tasks. Moved some files out of directory to sync in order to do basic sync test.
|
||||
|
||||
---
|
||||
[test@alma-azure-test AZURE]$ ./run_azcopy.sh /opt/AZURE/BLOB/ true 20
|
||||
Initial file list created: 1 files found.
|
||||
Running: azcopy sync "/opt/AZURE/BLOB/" "https://115.blob.core.windows.net/115?sv=2022-11-02&ss=bfqt&srt=sco&sp=rwlacupitfx&se=2025-02-20T18:39:49Z&st=2025-02-19T10:39:49Z&spr=https&sig=xxxx" --recursive --cap-mbps 20
|
||||
azcopy started with PID 8493
|
||||
INFO: Any empty folders will not be processed, because source and/or destination doesn't have full folder support
|
||||
|
||||
Job 9692ef9e-3872-f64c-5dbb-6f8fd8ad220e has started
|
||||
Log file is located at: /home/test/.azcopy/9692ef9e-3872-f64c-5dbb-6f8fd8ad220e.log
|
||||
|
||||
100.0 %, 1 Done, 0 Failed, 0 Pending, 1 Total, 2-sec Throughput (Mb/s): 9.1727
|
||||
|
||||
Job 9692ef9e-3872-f64c-5dbb-6f8fd8ad220e Summary
|
||||
Files Scanned at Source: 1
|
||||
Files Scanned at Destination: 2
|
||||
Elapsed Time (Minutes): 4.4351
|
||||
Number of Copy Transfers for Files: 1
|
||||
Number of Copy Transfers for Folder Properties: 0
|
||||
Total Number of Copy Transfers: 1
|
||||
Number of Copy Transfers Completed: 1
|
||||
Number of Copy Transfers Failed: 0
|
||||
Number of Deletions at Destination: 0
|
||||
Total Number of Bytes Transferred: 662880628
|
||||
Total Number of Bytes Enumerated: 662880628
|
||||
Final Job Status: Completed
|
||||
|
||||
Completion report generated: completion_report_20250220_132836.txt
|
||||
|
||||
#4th test.
|
||||
|
||||
Changed busines hours to test localy second time going into the restricted window to check if it will stopp the process. Window 14-15 so it should stop at 14 and then resume at 15.
|
||||
We will use local execution to test if it stopps and then will test cron if it resumes as it should.
|
||||
|
||||
---
|
||||
[test@alma-azure-test AZURE]$ ./run_azcopy.sh /opt/AZURE/BLOB/ true 20
|
||||
Initial file list created: 4 files found.
|
||||
Running: azcopy sync "/opt/AZURE/BLOB/" "https://115.blob.core.windows.net/115?sv=2022-11-02&ss=bfqt&srt=sco&sp=rwlacupitfx&se=2025-02-20T18:39:49Z&st=2025-02-19T10:39:49Z&spr=https&sig=xxxx" --recursive --cap-mbps 20
|
||||
azcopy started with PID 8654
|
||||
INFO: Any empty folders will not be processed, because source and/or destination doesn't have full folder support
|
||||
|
||||
Job 13c1945e-3c6e-dd42-49c0-2c6722ced7ec has started
|
||||
Log file is located at: /home/test/.azcopy/13c1945e-3c6e-dd42-49c0-2c6722ced7ec.log
|
||||
|
||||
54.7 %, 1 Done, 0 Failed, 2 Pending, 3 Total, 2-sec Throughput (Mb/s): 20.1759Business hours started! Stopping azcopy...
|
||||
./run_azcopy.sh: line 95: COMPLETED_FILES * 100 / TOTAL_FILES : division by 0 (error token is "TOTAL_FILES ")
|
||||
Completion report generated: completion_report_20250220_135033.txt
|
||||
|
||||
#5th test.
|
||||
|
||||
Changed edge case were 0 files were transfered or incorectly processed from the log creating a division by 0 error. added check if it is running still to ensure we do not spawn many proccesses.
|
||||
It should start via cron at 15 and then be re run every hour untill 7Am where it should detect restricted business hours window and do nothing.
|
||||
|
||||
Failed due to how cron and its enviroment work.
|
||||
|
||||
---
|
||||
Feb 20 15:25:02 alma-azure-test CROND[8956]: (test) CMD (/usr/bin/env bash -lc /opt/AZURE/run_azcopy.sh /opt/AZURE/BLOB/ true 20)
|
||||
Feb 20 15:25:02 alma-azure-test CROND[8954]: (test) CMDOUT (Usage: /opt/AZURE/run_azcopy.sh <directory> [log=true|false] [bandwidth_mbps])
|
||||
Feb 20 15:25:02 alma-azure-test CROND[8954]: (test) CMDEND (/usr/bin/env bash -lc /opt/AZURE/run_azcopy.sh /opt/AZURE/BLOB/ true 20)
|
||||
|
||||
#6th test.
|
||||
|
||||
modified the cron job definition to.
|
||||
|
||||
27 * * * * /usr/bin/env bash -lc '/opt/AZURE/run_azcopy.sh /opt/AZURE/BLOB/ true 20'
|
||||
|
||||
Should ensure proper execution in a users envirmont.
|
||||
|
||||
---
|
||||
That worked and its running in the background.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user