#!/usr/bin/env bash set -uo pipefail CONF=/etc/log-upload/secrets.env [ -f "$CONF" ] || { echo "no config" >&2; exit 1; } . "$CONF" LOG=/var/log/log-upload.log SPOOL=/var/spool/log-upload STATE=/var/lib/log-upload/uploaded.list mkdir -p "$SPOOL" "$(dirname "$STATE")" touch "$STATE" log() { echo "[$(date -uIs)] $*" >> "$LOG"; } NODE_NAME="${LOG_NODE_NAME:?LOG_NODE_NAME not set}" detect_ip() { local ip for src in https://api.ipify.org https://ifconfig.me https://icanhazip.com; do ip=$(curl -fsSL --max-time 5 "$src" 2>/dev/null | tr -d '[:space:]') [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] && { echo "$ip"; return 0; } done ip=$(hostname -I 2>/dev/null | awk '{print $1}') [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] && { echo "$ip"; return 0; } return 1 } NODE_IP=$(detect_ip) || { log "ERROR: не удалось определить публичный IP"; exit 1; } DATE=$(date -u -d yesterday +%F) DATE_PATH=${DATE//-/\/} S3_PREFIX="${DATE_PATH}/${NODE_NAME}__${NODE_IP}" MARKER="${DATE}__${NODE_NAME}__${NODE_IP}" if grep -Fxq "$MARKER" "$STATE"; then log "SKIP: уже выгружали за ${DATE} (${MARKER}) — повторно не льём" echo "already uploaded for ${DATE}" exit 0 fi log "═══ START upload v2 for ${NODE_NAME}@${NODE_IP} → ${S3_PREFIX} ═══" declare -a FILES pick_xray() { local base=$1 target=$2 if [ -f "/var/log/remnanode/${base}.1.gz" ] && [ -s "/var/log/remnanode/${base}.1.gz" ]; then FILES+=("/var/log/remnanode/${base}.1.gz:${target}.gz:1") elif [ -f "/var/log/remnanode/${base}.1" ] && [ -s "/var/log/remnanode/${base}.1" ]; then FILES+=("/var/log/remnanode/${base}.1:${target}.gz:0") fi } pick_xray access.log xray-access.log pick_xray error.log xray-error.log pick_sys() { local base=$1 target=$2 if [ -f "/var/log/${base}.1.gz" ] && [ -s "/var/log/${base}.1.gz" ]; then FILES+=("/var/log/${base}.1.gz:${target}.gz:1") elif [ -f "/var/log/${base}.1" ] && [ -s "/var/log/${base}.1" ]; then FILES+=("/var/log/${base}.1:${target}.gz:0") elif [ -f "/var/log/${base}" ] && [ -s "/var/log/${base}" ]; then FILES+=("/var/log/${base}:${target}.gz:0") fi } pick_sys fail2ban.log fail2ban.log if [ ${#FILES[@]} -eq 0 ]; then log " WARN: no log files found" fi log " found ${#FILES[@]} log files" META_FILES="[" for ENTRY in "${FILES[@]}"; do SRC="${ENTRY%%:*}" rest="${ENTRY#*:}" S3_NAME="${rest%%:*}" PRE_GZ="${rest##*:}" DST="$SPOOL/${S3_NAME}" if [ "$PRE_GZ" = "1" ]; then cp --reflink=auto "$SRC" "$DST" 2>/dev/null || cp "$SRC" "$DST" log " picked ${SRC} → ${S3_NAME} (already gz, no recompress)" else gzip -c "$SRC" > "$DST" log " gzipped ${SRC} → ${S3_NAME}" fi SIZE=$(stat -c%s "$DST") SRC_SIZE=$(stat -c%s "$SRC") LINES=$(zcat -f "$SRC" 2>/dev/null | wc -l) SHA=$(sha256sum "$DST" | awk '{print $1}') META_FILES+="{\"name\":\"${S3_NAME}\",\"src_size\":${SRC_SIZE},\"size\":${SIZE},\"sha256\":\"${SHA}\",\"lines\":${LINES}}," done META_FILES="${META_FILES%,}]" cat > "$SPOOL/_meta.json" </dev/null | grep -q '^tw$'; then $MC alias set tw "$S3_ENDPOINT" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" --api S3v4 >>"$LOG" 2>&1 fi upload_with_retry() { local file=$1 key=$2 for attempt in 1 2 3 4 5; do if $MC cp --quiet "$file" "tw/${S3_BUCKET}/${key}" >>"$LOG" 2>&1; then log " uploaded → ${key}" return 0 fi log " upload FAIL attempt=$attempt for ${key}, retry in 30s" sleep 30 done log " ERROR: failed after 5 attempts: ${key}" return 1 } OK=0 FAIL=0 for f in "$SPOOL"/*.gz "$SPOOL"/_meta.json; do [ -f "$f" ] || continue BASE=$(basename "$f") if upload_with_retry "$f" "${S3_PREFIX}/${BASE}"; then OK=$((OK+1)) rm -f "$f" else FAIL=$((FAIL+1)) fi done if [ "$FAIL" -eq 0 ] && [ "$OK" -gt 0 ]; then echo "$MARKER" >> "$STATE" log " marker записан в $STATE" fi log "═══ DONE ok=$OK fail=$FAIL ═══" log "" echo "uploaded for ${DATE}: ok=${OK} fail=${FAIL}" exit "$FAIL"