#!/bin/bash
# Create FIFO for bidirectional communication
PIPE_TO_WORKER="/tmp/to_worker.$$"
PIPE_FROM_WORKER="/tmp/from_worker.$$"
mkfifo "$PIPE_TO_WORKER" "$PIPE_FROM_WORKER"
# Cleanup on exit
trap "rm -f $PIPE_TO_WORKER $PIPE_FROM_WORKER" EXIT
# Worker process
worker() {
while read command; do
case "$command" in
"ping")
echo "pong"
;;
"time")
date +%s
;;
"quit")
break
;;
*)
echo "unknown command: $command"
;;
esac
done < "$PIPE_TO_WORKER" > "$PIPE_FROM_WORKER"
}
# Start worker in background
worker &
WORKER_PID=$!
# Main process - send commands and read responses
exec 3> "$PIPE_TO_WORKER" # Open for writing
exec 4< "$PIPE_FROM_WORKER" # Open for reading
echo "ping" >&3
read -u 4 response
echo "Response: $response"
echo "time" >&3
read -u 4 timestamp
echo "Timestamp: $timestamp"
echo "quit" >&3
# Close file descriptors
exec 3>&-
exec 4<&-
wait "$WORKER_PID"
#!/bin/bash
# Process multiple log files in parallel
process_logs() {
local logfile=$1
echo "Processing $logfile..."
grep "ERROR" "$logfile" | wc -l
}
# Export function for subshells
export -f process_logs
# Process all logs in parallel and collect results
paste \
<(process_logs /var/log/app1.log) \
<(process_logs /var/log/app2.log) \
<(process_logs /var/log/app3.log) \
| awk '{print "Total errors:", $1+$2+$3}'
#!/bin/bash
# Track child processes
declare -A CHILD_PIDS
# Cleanup function
cleanup() {
echo "Cleaning up..."
for pid in "${CHILD_PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
echo "Killing child process $pid"
kill "$pid"
fi
done
}
# Signal handlers
trap cleanup EXIT
trap 'echo "Received SIGTERM"; cleanup; exit 143' TERM
trap 'echo "Received SIGINT"; cleanup; exit 130' INT
# Ignore certain signals
trap '' HUP
# Forward signals to children
forward_signal() {
local signal=$1
for pid in "${CHILD_PIDS[@]}"; do
kill -"$signal" "$pid" 2>/dev/null
done
}
trap 'forward_signal USR1' USR1
trap 'forward_signal USR2' USR2
# Start child processes
for i in {1..3}; do
(
trap 'echo "Child $i received USR1"' USR1
trap 'echo "Child $i received USR2"' USR2
sleep 300
) &
CHILD_PIDS[$i]=$!
done
# Wait for children
wait
#!/bin/bash
# Use /dev/shm for fast inter-process data sharing
SHM_FILE="/dev/shm/myapp_$$"
# Writer process
writer() {
for i in {1..100}; do
echo "Message $i: $(date +%s.%N)" >> "$SHM_FILE"
sleep 0.1
done
}
# Reader process
reader() {
tail -f "$SHM_FILE" | while read line; do
echo "Read: $line"
# Process data
done
}
# Cleanup
trap "rm -f $SHM_FILE" EXIT
# Start processes
writer &
WRITER_PID=$!
sleep 1
reader &
READER_PID=$!
wait "$WRITER_PID"
kill "$READER_PID"
#!/bin/bash
# File-based locking
LOCKFILE="/var/lock/myapp.lock"
LOCKFD=200
# Acquire lock
acquire_lock() {
local timeout=${1:-10}
local waited=0
# Open file descriptor
eval "exec $LOCKFD>$LOCKFILE"
while ! flock -n $LOCKFD; do
if [ $waited -ge $timeout ]; then
echo "Failed to acquire lock after ${timeout}s" >&2
return 1
fi
sleep 1
((waited++))
done
# Write PID to lockfile
echo $$ >&$LOCKFD
return 0
}
# Release lock
release_lock() {
flock -u $LOCKFD
eval "exec $LOCKFD>&-"
rm -f "$LOCKFILE"
}
# Usage
if acquire_lock 30; then
trap release_lock EXIT
# Critical section
echo "Processing..."
sleep 5
# Lock automatically released on exit
else
echo "Another instance is running"
exit 1
fi
# Alternative: Use directory as lock (atomic operation)
atomic_lock() {
local lockdir="/var/lock/myapp.lock"
local max_attempts=30
for ((i=0; i<max_attempts; i++)); do
if mkdir "$lockdir" 2>/dev/null; then
trap "rmdir $lockdir" EXIT
return 0
fi
sleep 1
done
return 1
}
#!/bin/bash
# Simple HTTP GET request
http_get() {
local host=$1
local port=${2:-80}
local path=${3:-/}
# Open connection
exec 3<> /dev/tcp/"$host"/"$port"
# Send HTTP request
echo -e "GET $path HTTP/1.1\r\nHost: $host\r\nConnection: close\r\n\r\n" >&3
# Read response
while read -r line <&3; do
echo "$line"
done
# Close connection
exec 3>&-
}
# Usage
http_get "example.com" 80 "/"
# POST request
http_post() {
local host=$1
local port=$2
local path=$3
local data=$4
exec 3<> /dev/tcp/"$host"/"$port"
local content_length=${#data}
cat >&3 << EOF
POST $path HTTP/1.1
Host: $host
Content-Type: application/x-www-form-urlencoded
Content-Length: $content_length
Connection: close
$data
EOF
cat <&3
exec 3>&-
}
# Simple TCP client
tcp_send() {
local host=$1
local port=$2
local message=$3
exec 3<> /dev/tcp/"$host"/"$port"
echo "$message" >&3
cat <&3
exec 3>&-
}
# Check if port is open
port_open() {
local host=$1
local port=$2
local timeout=${3:-2}
timeout "$timeout" bash -c "cat < /dev/null > /dev/tcp/$host/$port" 2>/dev/null
return $?
}
# Usage
if port_open "example.com" 443 2; then
echo "Port 443 is open"
else
echo "Port 443 is closed"
fi
#!/bin/bash
# Very simple HTTP server using netcat
HTTP_PORT=8080
WEB_ROOT="/var/www/html"
http_server() {
local port=$1
local docroot=$2
while true; do
# Read HTTP request
REQUEST=$(nc -l -p "$port" | head -n 1)
METHOD=$(echo "$REQUEST" | cut -d' ' -f1)
PATH=$(echo "$REQUEST" | cut -d' ' -f2)
# Remove leading slash and handle root
FILE="${PATH#/}"
[ -z "$FILE" ] && FILE="index.html"
FILEPATH="$docroot/$FILE"
if [ -f "$FILEPATH" ]; then
# Determine content type
case "${FILE##*.}" in
html|htm) CONTENT_TYPE="text/html" ;;
css) CONTENT_TYPE="text/css" ;;
js) CONTENT_TYPE="application/javascript" ;;
json) CONTENT_TYPE="application/json" ;;
jpg|jpeg) CONTENT_TYPE="image/jpeg" ;;
png) CONTENT_TYPE="image/png" ;;
*) CONTENT_TYPE="text/plain" ;;
esac
CONTENT=$(<"$FILEPATH")
LENGTH=${#CONTENT}
# Send response
{
echo -e "HTTP/1.1 200 OK\r"
echo -e "Content-Type: $CONTENT_TYPE\r"
echo -e "Content-Length: $LENGTH\r"
echo -e "Connection: close\r"
echo -e "\r"
echo -n "$CONTENT"
} | nc -l -p "$port"
else
# 404 response
{
echo -e "HTTP/1.1 404 Not Found\r"
echo -e "Content-Type: text/html\r"
echo -e "Connection: close\r"
echo -e "\r"
echo -e "<h1>404 Not Found</h1>\r"
} | nc -l -p "$port"
fi
sleep 0.1
done
}
# Alternative using socat
http_server_socat() {
socat TCP-LISTEN:8080,reuseaddr,fork \
EXEC:'/bin/bash -c "
read -r request;
path=\$(echo \$request | cut -d\" \" -f2);
file=/var/www/html\${path};
if [ -f \$file ]; then
echo -e \"HTTP/1.1 200 OK\\r\";
echo -e \"\\r\";
cat \$file;
else
echo -e \"HTTP/1.1 404 Not Found\\r\";
echo -e \"\\r\";
echo \"404 Not Found\";
fi
"'
}
#!/bin/bash
# Advanced curl wrapper for REST APIs
API_BASE_URL="https://api.example.com"
API_TOKEN=""
# Set authentication
set_auth() {
API_TOKEN=$1
}
# Generic API request
api_request() {
local method=$1
local endpoint=$2
local data=${3:-}
local url="${API_BASE_URL}${endpoint}"
local headers=(
-H "Authorization: Bearer $API_TOKEN"
-H "Content-Type: application/json"
-H "Accept: application/json"
)
local curl_opts=(
-s
-w "\n%{http_code}"
-X "$method"
"${headers[@]}"
)
if [ -n "$data" ]; then
curl_opts+=(-d "$data")
fi
local response
response=$(curl "${curl_opts[@]}" "$url")
local http_code
http_code=$(echo "$response" | tail -n 1)
local body
body=$(echo "$response" | sed '$d')
if [ "$http_code" -ge 200 ] && [ "$http_code" < 300 ]; then
echo "$body" | jq '.' 2>/dev/null || echo "$body"
return 0
else
echo "API Error ($http_code): $body" >&2
return 1
fi
}
# Convenience functions
api_get() { api_request GET "$1"; }
api_post() { api_request POST "$1" "$2"; }
api_put() { api_request PUT "$1" "$2"; }
api_delete() { api_request DELETE "$1"; }
# Usage examples
set_auth "your-api-token-here"
# GET request
api_get "/users/123"
# POST request with JSON data
api_post "/users" '{
"name": "John Doe",
"email": "john@example.com"
}'
# PUT request
api_put "/users/123" '{
"name": "Jane Doe"
}'
# DELETE request
api_delete "/users/123"
# Pagination handling
api_get_all_pages() {
local endpoint=$1
local page=1
local results=()
while true; do
local response
response=$(api_get "${endpoint}?page=${page}&per_page=100")
if [ -z "$response" ]; then
break
fi
results+=("$response")
# Check if there are more pages (example for GitHub API)
local has_next
has_next=$(echo "$response" | jq 'length')
if [ "$has_next" < 100 ]; then
break
fi
((page++))
done
# Merge all results
printf '%s\n' "${results[@]}" | jq -s 'add'
}
# Rate limiting with exponential backoff
api_request_with_retry() {
local max_retries=5
local retry_count=0
local backoff=1
while [ $retry_count < $max_retries ]; do
local response
if response=$(api_request "$@" 2>&1); then
echo "$response"
return 0
fi
# Check if rate limited (HTTP 429)
if echo "$response" | grep -q "429"; then
echo "Rate limited, waiting ${backoff}s..." >&2
sleep "$backoff"
backoff=$((backoff * 2))
((retry_count++))
else
return 1
fi
done
echo "Max retries exceeded" >&2
return 1
}
#!/bin/bash
# WebSocket client example
ws_connect() {
local url=$1
# Install websocat if not available:
# cargo install websocat
# or download binary
websocat "$url"
}
# WebSocket with custom logic
ws_client() {
local url=$1
{
# Send messages
echo '{"type": "subscribe", "channel": "trades"}'
sleep 1
echo '{"type": "ping"}'
} | websocat "$url" | while read -r msg; do
# Process received messages
echo "Received: $msg"
# Parse JSON and act on it
local msg_type
msg_type=$(echo "$msg" | jq -r '.type')
case "$msg_type" in
"pong")
echo "Pong received"
;;
"trade")
# Process trade data
local price
price=$(echo "$msg" | jq -r '.price')
echo "Trade at price: $price"
;;
esac
done
}
# Usage
# ws_client "wss://stream.example.com/ws"
#!/bin/bash
# AWK script for log analysis
awk '
BEGIN {
# Initialize
FS = " "
errors = 0
warnings = 0
# Associative arrays
by_hour[""] = 0
by_status[""] = 0
}
{
# Extract timestamp (assuming format: [DD/Mon/YYYY:HH:MM:SS +0000])
if (match($4, /\[([0-9]+)\/([A-Z][a-z]+)\/([0-9]+):([0-9]+):([0-9]+):([0-9]+)/, arr)) {
hour = arr[4]
by_hour[hour]++
}
# Count by status code
status = $9
by_status[status]++
# Count error types
if (status >= 400 && status < 500) {
errors++
}
if (status >= 500) {
warnings++
}
# Calculate response size
size = $10
if (size ~ /^[0-9]+$/) {
total_bytes += size
request_count++
}
}
END {
print "=== Summary ==="
print "Total requests:", NR
print "Errors (4xx):", errors
print "Server errors (5xx):", warnings
if (request_count > 0) {
print "Average response size:", total_bytes / request_count, "bytes"
}
print "\n=== Requests by Hour ==="
for (hour in by_hour) {
if (hour != "") {
printf "%02d:00 - %d requests\n", hour, by_hour[hour]
}
}
print "\n=== Requests by Status ==="
for (status in by_status) {
if (status != "") {
printf "HTTP %s: %d requests\n", status, by_status[status]
}
}
}
' /var/log/apache2/access.log
# AWK functions
awk '
# Define functions
function abs(x) {
return x < 0 ? -x : x
}
function min(a, b) {
return a < b ? a : b
}
function max(a, b) {
return a > b ? a : b
}
# Use functions
{
print "Value:", $1, "Absolute:", abs($1)
print "Min of", $1, "and", $2, "is", min($1, $2)
}
' data.txt
# Multi-file processing
awk '
# FILENAME gives current file name
# FNR is line number in current file
# NR is total line number across all files
FNR == 1 {
print "Processing file:", FILENAME
}
{
# Count lines per file
file_lines[FILENAME]++
}
END {
for (file in file_lines) {
print file ":", file_lines[file], "lines"
}
}
' file1.txt file2.txt file3.txt
#!/bin/bash
# Hold space and pattern space manipulation
sed '
# Copy pattern space to hold space
h
# Append pattern space to hold space
H
# Copy hold space to pattern space
g
# Append hold space to pattern space
G
# Exchange pattern and hold space
x
' file.txt
# Reverse lines using hold space
sed '1!G;h;$!d' file.txt
# Remove duplicate consecutive lines
sed '$!N; /^\(.*\)\n\1$/!P; D' file.txt
# Add line numbers with formatting
sed = file.txt | sed 'N;s/\n/\t/'
# Multi-line pattern matching
sed '
# Join lines ending with backslash
:a
/\\$/ {
N
s/\\\n//
ta
}
' file.txt
# Advanced substitution with backreferences
sed '
# Swap two fields
s/^\([^,]*\),\([^,]*\)/\2,\1/
# Capture and reuse
s/\([0-9]\{3\}\)-\([0-9]\{3\}\)-\([0-9]\{4\}\)/(\1) \2-\3/
' file.txt
# Conditional processing
sed '
# Apply substitution only if line matches pattern
/pattern/s/old/new/g
# Range-based operations
/START/,/END/ {
s/foo/bar/g
}
# Negation
/pattern/!d # Delete all lines not matching pattern
' file.txt
# Insert file contents at pattern
sed '/pattern/r insert.txt' file.txt
# Write matching lines to file
sed '/pattern/w output.txt' file.txt
# Multiple scripts
sed -e 's/foo/bar/' -e 's/baz/qux/' file.txt
# Script from file
sed -f script.sed file.txt
#!/bin/bash
# In-place editing with backup
perl -i.bak -pe 's/old/new/g' file.txt
# JSON parsing
echo '{"name":"John","age":30}' | perl -MJSON -0777 -ne '
$data = decode_json($_);
print "$data->{name} is $data->{age} years old\n";
'
# URL encoding
echo "hello world" | perl -MURI::Escape -ne 'print uri_escape($_)'
# Base64 encoding/decoding
echo "hello" | perl -MMIME::Base64 -ne 'print encode_base64($_)'
echo "aGVsbG8K" | perl -MMIME::Base64 -ne 'print decode_base64($_)'
# CSV parsing
perl -MText::CSV -ne '
BEGIN { $csv = Text::CSV->new({ sep_char => "," }) }
$csv->parse($_);
@fields = $csv->fields();
print "Column 1: $fields[0], Column 2: $fields[1]\n";
' data.csv
# Advanced regex with lookahead/lookbehind
perl -ne 'print if /(?<=prefix).*?(?=suffix)/' file.txt
# Calculate statistics
perl -MList::Util=sum,max,min -ane '
END {
$sum = sum(@numbers);
$count = scalar(@numbers);
$avg = $sum / $count;
print "Sum: $sum, Avg: $avg, Max: ", max(@numbers), "\n";
}
push @numbers, $F[0];
' numbers.txt
# XML parsing
perl -MXML::Simple -e '
$xml = XMLin("file.xml");
print $xml->{element}->{subelement};
'
# Date manipulation
perl -MDateTime -e '
$dt = DateTime->now();
print $dt->ymd, " ", $dt->hms, "\n";
$tomorrow = $dt->add(days => 1);
print "Tomorrow: ", $tomorrow->ymd, "\n";
'
#!/bin/bash
# Email validation
validate_email() {
local email=$1
local regex='^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
[[ $email =~ $regex ]]
}
# URL validation
validate_url() {
local url=$1
local regex='^(https?|ftp)://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(/.*)?$'
[[ $url =~ $regex ]]
}
# IP address validation
validate_ipv4() {
local ip=$1
local regex='^([0-9]{1,3}\.){3}[0-9]{1,3}$'
if [[ $ip =~ $regex ]]; then
IFS='.' read -ra OCTETS <<< "$ip"
for octet in "${OCTETS[@]}"; do
if [ "$octet" -gt 255 ]; then
return 1
fi
done
return 0
fi
return 1
}
# Phone number extraction and formatting
format_phone() {
local input=$1
# Remove all non-digits
local digits=$(echo "$input" | tr -cd '0-9')
# Format as (XXX) XXX-XXXX
if [ ${#digits} -eq 10 ]; then
echo "${digits:0:3}-${digits:3:3}-${digits:6:4}"
else
echo "Invalid phone number"
return 1
fi
}
# Extract all URLs from text
extract_urls() {
local file=$1
grep -Eo 'https?://[a-zA-Z0-9./?=_-]*' "$file"
}
# Extract all email addresses
extract_emails() {
local file=$1
grep -Eo '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' "$file"
}
# Parse log timestamps
parse_log_timestamp() {
local line=$1
# Apache log format: [DD/Mon/YYYY:HH:MM:SS +0000]
if [[ $line =~ \[([0-9]{2})/([A-Z][a-z]{2})/([0-9]{4}):([0-9]{2}):([0-9]{2}):([0-9]{2})\ ([+-][0-9]{4})\] ]]; then
local day=${BASH_REMATCH[1]}
local month=${BASH_REMATCH[2]}
local year=${BASH_REMATCH[3]}
local hour=${BASH_REMATCH[4]}
local min=${BASH_REMATCH[5]}
local sec=${BASH_REMATCH[6]}
local tz=${BASH_REMATCH[7]}
echo "Date: $year-$month-$day $hour:$min:$sec $tz"
fi
}
# Multi-line pattern extraction
extract_xml_tag() {
local file=$1
local tag=$2
perl -0777 -ne "print \"\$1\n\" while /<$tag>(.*?)<\/$tag>/gs" "$file"
}
# Extract code blocks from markdown
extract_code_blocks() {
local file=$1
awk '
/^```/ {
in_block = !in_block
next
}
in_block {
print
}
' "$file"
}
#!/bin/bash
# Simple timing
time_command() {
local start=$(date +%s.%N)
"$@"
local end=$(date +%s.%N)
local elapsed=$(echo "$end - $start" | bc)
echo "Elapsed time: ${elapsed}s" >&2
}
# Usage
time_command ls -la /usr/bin
# Detailed benchmarking
benchmark() {
local iterations=${1:-10}
shift
local times=()
for ((i=0; i<iterations; i++)); do
local start=$(date +%s.%N)
"$@" > /dev/null 2>&1
local end=$(date +%s.%N)
local elapsed=$(echo "$end - $start" | bc)
times+=("$elapsed")
done
# Calculate statistics
local sum=0
local min=${times[0]}
local max=${times[0]}
for time in "${times[@]}"; do
sum=$(echo "$sum + $time" | bc)
if (( $(echo "$time < $min" | bc -l) )); then
min=$time
fi
if (( $(echo "$time > $max" | bc -l) )); then
max=$time
fi
done
local avg=$(echo "scale=6; $sum / $iterations" | bc)
echo "Iterations: $iterations"
echo "Average: ${avg}s"
echo "Min: ${min}s"
echo "Max: ${max}s"
}
# Compare two implementations
compare_implementations() {
local name1=$1
local name2=$2
local iterations=${3:-100}
shift 3
local impl1=("${@:1:$#/2}")
local impl2=("${@:$#/2+1}")
echo "Benchmarking $name1..."
local time1
time1=$(benchmark "$iterations" "${impl1[@]}" | grep "Average:" | awk '{print $2}')
echo "Benchmarking $name2..."
local time2
time2=$(benchmark "$iterations" "${impl2[@]}" | grep "Average:" | awk '{print $2}')
# Calculate difference
local diff=$(echo "scale=2; (($time2 - $time1) / $time1) * 100" | bc)
echo "Results:"
echo " $name1: ${time1}s"
echo " $name2: ${time2}s"
echo " Difference: ${diff}%"
}
#!/bin/bash
# Enable profiling
PS4='+ $(date "+%s.%N")\011 '
exec 3>&2 2>/tmp/bashstart.$$.log
set -x
# Your script here
for i in {1..100}; do
echo "Processing $i"
sleep 0.01
done
# Disable profiling
set +x
exec 2>&3 3>&-
# Analyze profile
awk '
{
# Extract timestamp and command
if (NF >= 2) {
cmd = $2
for (i=3; i<=NF; i++) cmd = cmd " " $i
if (last_time != "") {
diff = $1 - last_time
times[last_cmd] += diff
counts[last_cmd]++
}
last_time = $1
last_cmd = cmd
}
}
END {
print "Command execution times:"
for (cmd in times) {
printf "%10.6f %5d %s\n", times[cmd], counts[cmd], cmd
}
}
' /tmp/bashstart.$$.log | sort -rn | head -20
rm /tmp/bashstart.$$.log
#!/bin/bash
# Avoid loading entire file into memory
# ❌ BAD: Loads entire file
CONTENT=$(cat huge_file.txt)
echo "$CONTENT" | process_line_by_line
# ✓ GOOD: Process line by line
while IFS= read -r line; do
process_line "$line"
done < huge_file.txt
# Use mapfile for arrays efficiently
# ❌ BAD: Slow for large files
LINES=()
while IFS= read -r line; do
LINES+=("$line")
done < file.txt
# ✓ GOOD: Much faster
mapfile -t LINES < file.txt
# Stream processing
# ❌ BAD: Multiple passes
grep "pattern1" file.txt > /tmp/temp1
grep "pattern2" /tmp/temp1 > /tmp/temp2
wc -l /tmp/temp2
# ✓ GOOD: Single pass
grep "pattern1" file.txt | grep "pattern2" | wc -l
# Memory-efficient sorting
# For huge files, use external sort
sort -T /tmp --parallel=4 huge_file.txt
# Limit memory usage
ulimit -v 1000000 # Limit to ~1GB virtual memory
#!/bin/bash
# Use built-ins instead of external commands
# ❌ SLOW
result=$(expr 5 + 3)
length=$(echo "$string" | wc -c)
# ✓ FAST
result=$((5 + 3))
length=${#string}
# String operations
# ❌ SLOW
basename=$(echo "$path" | awk -F/ '{print $NF}')
# ✓ FAST
basename=${path##*/}
# Case conversion
# ❌ SLOW
upper=$(echo "$string" | tr '[:lower:]' '[:upper:]')
# ✓ FAST (BASH 4+)
upper=${string^^}
# Pattern matching
# ❌ SLOW (spawns grep process)
if echo "$string" | grep -q "pattern"; then
echo "found"
fi
# ✓ FAST (built-in)
if [[ $string == *"pattern"* ]]; then
echo "found"
fi
# Array operations
# ✓ Efficient array iteration
for item in "${array[@]}"; do
process "$item"
done
# ✓ Efficient array slicing
subset=("${array[@]:start:length}")
# Parallel processing for independent tasks
# Sequential
for file in *.txt; do
process_file "$file"
done
# Parallel (using GNU parallel)
parallel process_file ::: *.txt
# Parallel (using xargs)
ls *.txt | xargs -P 4 -I {} process_file {}
# Parallel (manual with background jobs)
MAX_JOBS=4
for file in *.txt; do
while [ $(jobs -r | wc -l) -ge $MAX_JOBS ]; do
sleep 0.1
done
process_file "$file" &
done
wait
#!/bin/bash # BASH is great for: # - System administration tasks # - Command orchestration # - File operations # - Simple text processing # - Prototyping # Use Python/Perl/Ruby when you need: # - Complex data structures (beyond arrays) # - Object-oriented programming # - Heavy string/regex processing # - Math operations # - JSON/XML parsing (though jq works well) # - Web scraping # - Database operations # - GUI applications # Performance comparison example: # For a task processing 1M records: # - BASH with awk: ~30 seconds # - Python with pandas: ~5 seconds # - C program: ~1 second # Rule of thumb: # - Script > 500 lines? Consider another language # - Complex algorithms? Not BASH # - Need maintainability? Document well or use Python # - Performance critical? Profile first, optimize or switch language
#!/bin/bash
# Call Python from BASH
python3 << 'EOF'
import sys
import json
data = {"name": "John", "age": 30}
print(json.dumps(data, indent=2))
EOF
# Pass variables to Python
NAME="John"
AGE=30
python3 << EOF
name = "$NAME"
age = $AGE
print(f"{name} is {age} years old")
# Complex calculations
import math
result = math.sqrt($AGE)
print(f"Square root of {age}: {result}")
EOF
# Capture Python output
RESULT=$(python3 -c "
import json
data = {'items': [1, 2, 3, 4, 5]}
print(json.dumps(data))
")
echo "Python result: $RESULT"
# Use Python for JSON parsing
parse_json() {
local json=$1
local key=$2
python3 << EOF
import json
import sys
data = json.loads('$json')
print(data.get('$key', ''))
EOF
}
JSON='{"name":"John","age":30}'
NAME=$(parse_json "$JSON" "name")
echo "Name: $NAME"
# Python virtual environment in BASH
setup_python_env() {
python3 -m venv venv
source venv/bin/activate
pip install requests beautifulsoup4
}
# Web scraping with Python from BASH
scrape_url() {
local url=$1
python3 << EOF
import requests
from bs4 import BeautifulSoup
response = requests.get('$url')
soup = BeautifulSoup(response.text, 'html.parser')
# Extract titles
for title in soup.find_all('h1'):
print(title.get_text())
EOF
}
#!/bin/bash
# Compile and use C program from BASH
create_c_helper() {
cat > /tmp/helper.c << 'EOF'
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// Fast string search
int main(int argc, char *argv[]) {
if (argc != 3) {
fprintf(stderr, "Usage: %s <file> <pattern>\n", argv[0]);
return 1;
}
FILE *fp = fopen(argv[1], "r");
if (!fp) {
perror("fopen");
return 1;
}
char line[4096];
int line_num = 0;
int count = 0;
while (fgets(line, sizeof(line), fp)) {
line_num++;
if (strstr(line, argv[2])) {
printf("%d: %s", line_num, line);
count++;
}
}
fclose(fp);
printf("Found %d matches\n", count);
return 0;
}
EOF
gcc -O3 -o /tmp/helper /tmp/helper.c
}
# Use the C program
create_c_helper
/tmp/helper /var/log/syslog "error"
# Shared library example
create_shared_lib() {
cat > /tmp/mathlib.c << 'EOF'
#include <math.h>
double fast_sqrt(double x) {
return sqrt(x);
}
int fast_power(int base, int exp) {
int result = 1;
while (exp > 0) {
if (exp % 2 == 1) result *= base;
base *= base;
exp /= 2;
}
return result;
}
EOF
gcc -shared -fPIC -o /tmp/libmath.so /tmp/mathlib.c -lm
}
# Call shared library from Python/other languages
create_shared_lib
#!/bin/bash
# BASH -> Python -> AWK -> BASH pipeline
analyze_logs() {
local logfile=$1
# BASH: Extract relevant lines
grep "ERROR" "$logfile" |
# Python: Parse and transform JSON logs
python3 -c '
import sys
import json
for line in sys.stdin:
try:
data = json.loads(line)
print(f"{data[\"timestamp\"]},{data[\"level\"]},{data[\"message\"]}")
except:
pass
' |
# AWK: Aggregate by hour
awk -F, '
{
split($1, dt, "T")
split(dt[2], time, ":")
hour = time[1]
by_hour[hour]++
}
END {
for (h in by_hour) {
print h "00", by_hour[h]
}
}' | sort
}
# Use multiple languages for their strengths
complex_pipeline() {
# BASH: File operations
find /var/log -name "*.log" -mtime -1 |
# Python: Complex filtering
python3 << 'EOF' |
import sys
import os
for line in sys.stdin:
filepath = line.strip()
# Complex logic here
if os.path.getsize(filepath) > 1000000:
print(filepath)
EOF
# BASH: Parallel processing
xargs -P 4 -I {} bash -c 'gzip {}'
}
#!/bin/bash
# Open file descriptor for reading
exec 3< input.txt
# Read from fd 3
while read -u 3 line; do
echo "Read: $line"
done
# Close fd 3
exec 3<&-
# Open file descriptor for writing
exec 4> output.txt
# Write to fd 4
echo "Line 1" >&4
echo "Line 2" >&4
# Close fd 4
exec 4>&-
# Read and write to same file
exec 5<> file.txt
# Read
read -u 5 first_line
echo "First line: $first_line"
# Write
echo "New line" >&5
# Close
exec 5>&-
# Multiple file descriptors
exec 3< file1.txt
exec 4< file2.txt
exec 5> output.txt
# Merge two files
while read -u 3 line1 && read -u 4 line2; do
echo "$line1 | $line2" >&5
done
exec 3<&-
exec 4<&-
exec 5>&-
#!/bin/bash
# Create bidirectional communication with process
bidirectional_pipe() {
local cmd=$1
# Create named pipes
local pipe_to="/tmp/pipe_to_$$"
local pipe_from="/tmp/pipe_from_$$"
mkfifo "$pipe_to" "$pipe_from"
# Start process with pipes
$cmd < "$pipe_to" > "$pipe_from" &
local pid=$!
# Open pipes for I/O
exec 3> "$pipe_to"
exec 4< "$pipe_from"
# Cleanup function
cleanup() {
exec 3>&-
exec 4<&-
kill $pid 2>/dev/null
rm -f "$pipe_to" "$pipe_from"
}
trap cleanup EXIT
# Send and receive data
echo "command 1" >&3
read -u 4 response1
echo "Response 1: $response1"
echo "command 2" >&3
read -u 4 response2
echo "Response 2: $response2"
}
# Usage with bc calculator
bidirectional_calc() {
exec 3> >(bc -l)
exec 4< <(bc -l)
echo "10 + 20" >&3
read -u 4 result
echo "Result: $result"
exec 3>&-
exec 4<&-
}
#!/bin/bash
# Swap stdout and stderr
swap_outputs() {
# Save original descriptors
exec 3>&1 # Save stdout to fd 3
exec 4>&2 # Save stderr to fd 4
# Swap them
exec 1>&4 # Redirect stdout to original stderr
exec 2>&3 # Redirect stderr to original stdout
# Now stdout and stderr are swapped
echo "This goes to original stderr"
echo "This also goes to original stderr" >&2 # Now goes to original stdout
# Restore
exec 1>&3 # Restore stdout
exec 2>&4 # Restore stderr
exec 3>&- # Close fd 3
exec 4>&- # Close fd 4
}
# Redirect everything to file and still see it
tee_all() {
local logfile=$1
# Redirect stdout and stderr to file and terminal
exec > >(tee -a "$logfile")
exec 2> >(tee -a "$logfile" >&2)
echo "This goes to terminal and file"
echo "This error too" >&2
}
# Create log with timestamp
timestamped_log() {
local logfile=$1
exec > >(while read -r line; do
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $line"
done | tee -a "$logfile")
exec 2>&1
echo "Message with timestamp"
}
# Descriptor table manipulation
show_open_fds() {
echo "Open file descriptors:"
ls -l /proc/$$/fd/
}
# Dynamic fd allocation
allocate_fd() {
local fd
# Find next available fd
for ((fd=10; fd<100; fd++)); do
if ! { true >&$fd; } 2>/dev/null; then
echo "$fd"
return 0
fi
done
return 1
}
# Usage
FD=$(allocate_fd)
eval "exec $FD> /tmp/dynamic.txt"
echo "Writing to dynamic fd" >&$FD
eval "exec $FD>&-"
#!/bin/bash
# Validate input types
validate_integer() {
local input=$1
if [[ ! $input =~ ^-?[0-9]+$ ]]; then
echo "Error: Not a valid integer" >&2
return 1
fi
echo "$input"
}
validate_alphanumeric() {
local input=$1
if [[ ! $input =~ ^[a-zA-Z0-9]+$ ]]; then
echo "Error: Must be alphanumeric" >&2
return 1
fi
echo "$input"
}
validate_filename() {
local input=$1
# Allow only safe characters in filenames
if [[ ! $input =~ ^[a-zA-Z0-9._-]+$ ]]; then
echo "Error: Invalid filename" >&2
return 1
fi
# Prevent directory traversal
if [[ $input == *..* ]]; then
echo "Error: Directory traversal detected" >&2
return 1
fi
echo "$input"
}
validate_email() {
local email=$1
local regex='^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
if [[ ! $email =~ $regex ]]; then
echo "Error: Invalid email" >&2
return 1
fi
echo "$email"
}
validate_url() {
local url=$1
# Only allow http/https
if [[ ! $url =~ ^https?:// ]]; then
echo "Error: Only HTTP/HTTPS URLs allowed" >&2
return 1
fi
echo "$url"
}
# Sanitize input for shell commands
sanitize_shell() {
local input=$1
# Remove dangerous characters
input=$(echo "$input" | tr -d '\n\r;|&$`\\!(){}[]<>')
# Or use printf %q for shell escaping
printf '%q' "$input"
}
# Sanitize for SQL (basic)
sanitize_sql() {
local input=$1
# Escape single quotes
echo "${input//\'/\'\'}"
}
# Whitelist validation
validate_whitelist() {
local input=$1
shift
local whitelist=("$@")
for item in "${whitelist[@]}"; do
if [ "$input" = "$item" ]; then
echo "$input"
return 0
fi
done
echo "Error: Value not in whitelist" >&2
return 1
}
# Usage
ALLOWED_ACTIONS=("start" "stop" "restart" "status")
ACTION=$(validate_whitelist "$USER_INPUT" "${ALLOWED_ACTIONS[@]}")
#!/bin/bash
# Never hardcode credentials!
# ❌ BAD
DB_PASSWORD="secret123"
# ✓ GOOD: Use environment variables
DB_PASSWORD="${DB_PASSWORD:-}"
if [ -z "$DB_PASSWORD" ]; then
echo "Error: DB_PASSWORD not set" >&2
exit 1
fi
# ✓ GOOD: Read from secure file
read_password() {
local password_file="/etc/myapp/db_password"
# Check file permissions
if [ ! -f "$password_file" ]; then
echo "Password file not found" >&2
return 1
fi
local perms
perms=$(stat -c %a "$password_file" 2>/dev/null || stat -f %A "$password_file")
if [ "$perms" != "600" ] && [ "$perms" != "400" ]; then
echo "Insecure password file permissions: $perms" >&2
return 1
fi
cat "$password_file"
}
# ✓ GOOD: Read from password manager
read_from_password_manager() {
# Example with pass (passwordstore.org)
pass show database/production/password
# Example with AWS Secrets Manager
aws secretsmanager get-secret-value \
--secret-id myapp/db/password \
--query SecretString \
--output text
# Example with HashiCorp Vault
vault kv get -field=password secret/database/prod
}
# ✓ GOOD: Prompt user
read_password_securely() {
local password
read -sp "Enter password: " password
echo >&2 # New line
echo "$password"
}
# Clear sensitive variables
clear_password() {
local var_name=$1
# Overwrite with random data
eval "$var_name='$(head -c 32 /dev/urandom | base64)'"
# Unset
unset "$var_name"
}
# Example usage
DB_PASSWORD=$(read_password)
# Use password
mysql -p"$DB_PASSWORD" ...
# Clear it
clear_password DB_PASSWORD
# Avoid passwords in process list
# ❌ BAD: Password visible in ps
mysql -p"secret" ...
# ✓ GOOD: Use config file or stdin
mysql --defaults-file=/etc/mysql/client.cnf ...
# Or
echo "password" | mysql --password=$(cat) ...
#!/bin/bash
# Secure temporary files
create_secure_tempfile() {
local tmpfile
tmpfile=$(mktemp) || {
echo "Failed to create temp file" >&2
return 1
}
# Set restrictive permissions immediately
chmod 600 "$tmpfile"
# Automatic cleanup
trap "shred -u $tmpfile 2>/dev/null; rm -f $tmpfile" EXIT
echo "$tmpfile"
}
# Secure file writing
write_secure_file() {
local file=$1
local content=$2
local tmpfile
tmpfile=$(mktemp) || return 1
# Write to temp file
echo "$content" > "$tmpfile"
# Set permissions before moving
chmod 600 "$tmpfile"
# Atomic move
mv "$tmpfile" "$file"
}
# Check file before operations
safe_file_operation() {
local file=$1
# Check if file exists
if [ ! -e "$file" ]; then
echo "File does not exist" >&2
return 1
fi
# Check if regular file
if [ ! -f "$file" ]; then
echo "Not a regular file" >&2
return 1
fi
# Check if symbolic link (potential security issue)
if [ -L "$file" ]; then
echo "Will not operate on symbolic link" >&2
return 1
fi
# Check ownership
local owner
owner=$(stat -c %U "$file" 2>/dev/null || stat -f %Su "$file")
if [ "$owner" != "$USER" ] && [ "$owner" != "root" ]; then
echo "File not owned by current user or root" >&2
return 1
fi
# Perform operation
cat "$file"
}
# Prevent TOCTOU (Time Of Check Time Of Use) attacks
# ❌ BAD: Race condition
if [ -f "$file" ]; then
cat "$file" # File could be changed between check and use
fi
# ✓ GOOD: Check and use atomically
cat "$file" 2>/dev/null || {
echo "File does not exist or cannot be read" >&2
exit 1
}
# Secure deletion
secure_delete() {
local file=$1
if [ -f "$file" ]; then
# Overwrite with random data multiple times
shred -vfz -n 10 "$file"
# Then remove
rm -f "$file"
fi
}
#!/bin/bash
# Check if running as root
require_root() {
if [ "$EUID" -ne 0 ]; then
echo "This script must be run as root" >&2
exit 1
fi
}
# Check if NOT running as root
require_non_root() {
if [ "$EUID" -eq 0 ]; then
echo "This script should not be run as root" >&2
exit 1
fi
}
# Drop privileges
drop_privileges() {
local target_user=$1
if [ "$EUID" -ne 0 ]; then
echo "Must be root to drop privileges" >&2
return 1
fi
# Run command as different user
su - "$target_user" -c "$@"
}
# Temporary privilege escalation with sudo
run_with_sudo() {
if [ "$EUID" -ne 0 ]; then
sudo "$@"
else
"$@"
fi
}
# Check specific capability
has_capability() {
local capability=$1
# Check with getcap
if command -v getcap >/dev/null; then
getcap "$0" | grep -q "$capability"
else
# Fallback: check if root
[ "$EUID" -eq 0 ]
fi
}
# Secure sudo usage
secure_sudo_check() {
# Verify sudo is available
if ! command -v sudo >/dev/null; then
echo "sudo not available" >&2
return 1
fi
# Check if user has sudo rights
if ! sudo -n true 2>/dev/null; then
echo "User does not have sudo privileges" >&2
return 1
fi
return 0
}
# Audit logging
audit_log() {
local action=$1
local user="${SUDO_USER:-$USER}"
local timestamp
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
logger -t "$(basename "$0")" "[$timestamp] User: $user, Action: $action"
# Also log to file
echo "[$timestamp] User: $user, Action: $action" >> /var/log/script_audit.log
}
#!/bin/bash
# Prevent command injection
# ❌ DANGEROUS: User input in eval
eval "ls $USER_INPUT"
# ❌ DANGEROUS: User input in system()
system "grep $USER_INPUT file.txt"
# ✓ SAFE: Use arrays and proper quoting
grep "$USER_INPUT" file.txt
# ✓ SAFE: Validate input first
if [[ $USER_INPUT =~ ^[a-zA-Z0-9_]+$ ]]; then
grep "$USER_INPUT" file.txt
else
echo "Invalid input" >&2
exit 1
fi
# Prevent SQL injection (when using mysql)
# ❌ BAD: String concatenation
mysql -e "SELECT * FROM users WHERE name = '$USER_INPUT'"
# ✓ GOOD: Use parameterized queries (via programming language)
# Or escape properly
ESCAPED=$(printf '%s' "$USER_INPUT" | sed "s/'/\\\\'/g")
mysql -e "SELECT * FROM users WHERE name = '$ESCAPED'"
# Prevent path traversal
safe_path() {
local base_dir=$1
local user_path=$2
# Resolve to absolute path
local real_path
real_path=$(realpath -m "$base_dir/$user_path")
# Verify it's under base_dir
if [[ ! $real_path == $base_dir/* ]]; then
echo "Path traversal detected" >&2
return 1
fi
echo "$real_path"
}
# Prevent header injection
safe_header() {
local value=$1
# Remove newlines and carriage returns
echo "$value" | tr -d '\r\n'
}
#!/usr/bin/env bats
# Install bats:
# git clone https://github.com/bats-core/bats-core.git
# cd bats-core && ./install.sh /usr/local
# test_script.bats
# Load the script to test
load '/path/to/script.sh'
# Setup runs before each test
setup() {
# Create temp directory
TEST_TEMP_DIR="$(mktemp -d)"
}
# Teardown runs after each test
teardown() {
# Clean up
rm -rf "$TEST_TEMP_DIR"
}
# Test cases
@test "addition works" {
result=$(add 2 3)
[ "$result" -eq 5 ]
}
@test "validate_email accepts valid email" {
run validate_email "user@example.com"
[ "$status" -eq 0 ]
}
@test "validate_email rejects invalid email" {
run validate_email "not-an-email"
[ "$status" -eq 1 ]
}
@test "file_exists returns true for existing file" {
touch "$TEST_TEMP_DIR/test.txt"
run file_exists "$TEST_TEMP_DIR/test.txt"
[ "$status" -eq 0 ]
}
@test "file_exists returns false for missing file" {
run file_exists "$TEST_TEMP_DIR/missing.txt"
[ "$status" -eq 1 ]
}
@test "output contains expected text" {
run some_function
[ "$status" -eq 0 ]
[[ "$output" == *"expected text"* ]]
}
@test "stderr contains error message" {
run error_function
[ "$status" -eq 1 ]
[[ "$stderr" == *"error"* ]]
}
# Skip test conditionally
@test "this test requires root" {
if [ "$EUID" -ne 0 ]; then
skip "requires root"
fi
run privileged_operation
[ "$status" -eq 0 ]
}
# Test timeout
@test "function completes within time limit" {
run timeout 5 slow_function
[ "$status" -eq 0 ]
}
# Run bats tests:
# bats test_script.bats
#!/bin/bash
# integration_tests.sh
# Test configuration
TEST_DB="test_database"
TEST_USER="test_user"
TEST_PASSWORD="test_password"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Test counter
TESTS_RUN=0
TESTS_PASSED=0
TESTS_FAILED=0
# Assert functions
assert_equals() {
local expected=$1
local actual=$2
local message=${3:-"Assertion failed"}
((TESTS_RUN++))
if [ "$expected" = "$actual" ]; then
echo -e "${GREEN}✓ PASS${NC}: $message"
((TESTS_PASSED++))
return 0
else
echo -e "${RED}✗ FAIL${NC}: $message"
echo " Expected: $expected"
echo " Got: $actual"
((TESTS_FAILED++))
return 1
fi
}
assert_not_equals() {
local expected=$1
local actual=$2
local message=${3:-"Assertion failed"}
((TESTS_RUN++))
if [ "$expected" != "$actual" ]; then
echo -e "${GREEN}✓ PASS${NC}: $message"
((TESTS_PASSED++))
return 0
else
echo -e "${RED}✗ FAIL${NC}: $message"
echo " Expected not: $expected"
echo " Got: $actual"
((TESTS_FAILED++))
return 1
fi
}
assert_file_exists() {
local file=$1
local message=${2:-"File should exist"}
((TESTS_RUN++))
if [ -f "$file" ]; then
echo -e "${GREEN}✓ PASS${NC}: $message"
((TESTS_PASSED++))
return 0
else
echo -e "${RED}✗ FAIL${NC}: $message"
echo " File not found: $file"
((TESTS_FAILED++))
return 1
fi
}
assert_command_succeeds() {
local message=$1
shift
((TESTS_RUN++))
if "$@" >/dev/null 2>&1; then
echo -e "${GREEN}✓ PASS${NC}: $message"
((TESTS_PASSED++))
return 0
else
echo -e "${RED}✗ FAIL${NC}: $message"
echo " Command failed: $*"
((TESTS_FAILED++))
return 1
fi
}
# Setup test environment
setup_test_env() {
echo "Setting up test environment..."
# Create test database
mysql -e "CREATE DATABASE IF NOT EXISTS $TEST_DB"
# Create test user
mysql -e "CREATE USER IF NOT EXISTS '$TEST_USER'@'localhost' IDENTIFIED BY '$TEST_PASSWORD'"
mysql -e "GRANT ALL ON $TEST_DB.* TO '$TEST_USER'@'localhost'"
# Initialize schema
mysql "$TEST_DB" < schema.sql
}
# Cleanup test environment
cleanup_test_env() {
echo "Cleaning up test environment..."
mysql -e "DROP DATABASE IF EXISTS $TEST_DB"
mysql -e "DROP USER IF EXISTS '$TEST_USER'@'localhost'"
}
# Test suite
test_database_connection() {
echo -e "\n${YELLOW}Testing database connection${NC}"
assert_command_succeeds "Can connect to database" \
mysql -u"$TEST_USER" -p"$TEST_PASSWORD" -e "SELECT 1" "$TEST_DB"
}
test_api_endpoints() {
echo -e "\n${YELLOW}Testing API endpoints${NC}"
# Start test server
./start_server.sh --port 8888 --env test &
SERVER_PID=$!
# Wait for server to start
sleep 2
# Test GET endpoint
local status
status=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8888/api/status)
assert_equals "200" "$status" "GET /api/status returns 200"
# Test POST endpoint
local response
response=$(curl -s -X POST -H "Content-Type: application/json" \
-d '{"name":"test"}' \
http://localhost:8888/api/items)
assert_equals "true" \
"$(echo "$response" | jq -r '.success')" \
"POST /api/items succeeds"
# Stop server
kill $SERVER_PID
}
test_file_processing() {
echo -e "\n${YELLOW}Testing file processing${NC}"
# Create test file
echo "test data" > /tmp/test_input.txt
# Process file
./process_file.sh /tmp/test_input.txt /tmp/test_output.txt
# Verify output
assert_file_exists /tmp/test_output.txt "Output file created"
local output
output=$(cat /tmp/test_output.txt)
assert_equals "TEST DATA" "$output" "Data processed correctly"
# Cleanup
rm -f /tmp/test_input.txt /tmp/test_output.txt
}
# Run all tests
run_all_tests() {
echo "Starting integration tests..."
setup_test_env
test_database_connection
test_api_endpoints
test_file_processing
cleanup_test_env
# Print summary
echo -e "\n========================================"
echo "Test Results"
echo "========================================"
echo "Tests run: $TESTS_RUN"
echo -e "${GREEN}Passed: $TESTS_PASSED${NC}"
echo -e "${RED}Failed: $TESTS_FAILED${NC}"
if [ $TESTS_FAILED -eq 0 ]; then
echo -e "\n${GREEN}All tests passed!${NC}"
exit 0
else
echo -e "\n${RED}Some tests failed!${NC}"
exit 1
fi
}
# Run tests
run_all_tests
#!/bin/bash
# .gitlab-ci.yml equivalent in bash
# ci_pipeline.sh
set -euo pipefail
# Configuration
PROJECT_NAME="myapp"
BUILD_DIR="build"
TEST_RESULTS_DIR="test-results"
# Logging
log() {
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
}
error() {
log "ERROR: $*" >&2
exit 1
}
# Stage: Lint
stage_lint() {
log "Running lint stage..."
# Shellcheck for bash scripts
find . -name "*.sh" -type f | while read -r script; do
log "Linting $script"
shellcheck "$script" || error "Shellcheck failed for $script"
done
# Check for common issues
if grep -r "TODO\|FIXME" --include="*.sh" .; then
log "WARNING: Found TODO/FIXME comments"
fi
log "Lint stage passed"
}
# Stage: Build
stage_build() {
log "Running build stage..."
mkdir -p "$BUILD_DIR"
# Compile if needed
if [ -f "Makefile" ]; then
make clean
make || error "Build failed"
fi
# Package application
tar czf "$BUILD_DIR/${PROJECT_NAME}-${CI_COMMIT_SHA}.tar.gz" \
--exclude=".git" \
--exclude="$BUILD_DIR" \
.
log "Build stage passed"
}
# Stage: Test
stage_test() {
log "Running test stage..."
mkdir -p "$TEST_RESULTS_DIR"
# Run unit tests
if [ -f "run_tests.sh" ]; then
./run_tests.sh || error "Tests failed"
fi
# Run bats tests if available
if command -v bats >/dev/null && [ -d "tests" ]; then
bats tests/*.bats --formatter junit > "$TEST_RESULTS_DIR/junit.xml"
fi
# Run integration tests
if [ -f "integration_tests.sh" ]; then
./integration_tests.sh || error "Integration tests failed"
fi
log "Test stage passed"
}
# Stage: Security Scan
stage_security() {
log "Running security scan..."
# Check for hardcoded secrets
if grep -rE "(password|secret|api_key|token)\s*=\s*['\"][^'\"]+['\"]" \
--include="*.sh" --include="*.conf" .; then
error "Potential hardcoded secrets found"
fi
# Check file permissions
find . -type f \( -name "*.sh" -o -name "*.key" -o -name "*.pem" \) \
-perm /go+w -ls | grep . && \
error "World-writable sensitive files found"
log "Security scan passed"
}
# Stage: Deploy
stage_deploy() {
log "Running deploy stage..."
local env=${1:-staging}
local artifact="$BUILD_DIR/${PROJECT_NAME}-${CI_COMMIT_SHA}.tar.gz"
case "$env" in
staging)
log "Deploying to staging..."
scp "$artifact" staging@staging-server:/tmp/
ssh staging@staging-server "cd /opt/myapp && tar xzf /tmp/$(basename "$artifact")"
ssh staging@staging-server "systemctl restart myapp"
;;
production)
log "Deploying to production..."
# Require manual approval in real CI/CD
scp "$artifact" prod@prod-server:/tmp/
ssh prod@prod-server "cd /opt/myapp && tar xzf /tmp/$(basename "$artifact")"
ssh prod@prod-server "systemctl restart myapp"
;;
*)
error "Unknown environment: $env"
;;
esac
log "Deploy stage passed"
}
# Main pipeline
main() {
local stage=${1:-all}
# Set environment variables (would come from CI system)
export CI_COMMIT_SHA=${CI_COMMIT_SHA:-$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")}
export CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-$(git branch --show-current 2>/dev/null || echo "unknown")}
log "Starting CI pipeline for $PROJECT_NAME"
log "Commit: $CI_COMMIT_SHA"
log "Branch: $CI_COMMIT_BRANCH"
case "$stage" in
lint)
stage_lint
;;
build)
stage_build
;;
test)
stage_test
;;
security)
stage_security
;;
deploy)
stage_deploy "${2:-staging}"
;;
all)
stage_lint
stage_build
stage_test
stage_security
log "All stages passed!"
;;
*)
error "Unknown stage: $stage"
;;
esac
}
main "$@"
You now have advanced knowledge of:
This guide covers professional-level BASH scripting for DevOps, automation, and systems programming.
Deep Dive into BASH - Advanced Topics
Professional-level scripting techniques