syncrepo 3.64 KB
Newer Older
1
#!/bin/bash
Florian Pritz's avatar
Florian Pritz committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
########
#
# Copyright © 2014-2019 Florian Pritz <bluewind@xinu.at>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
#
########
#
22

23
24
25
26
27
# NOTE: This script is a copy of the syncrepo-template.sh script in the same
# directory.  It is a copy instead of being a template because the template is
# linked in the wiki. People that follow the link and want to use the script
# should populate the variables themselves and not be influenced by the values we
# are using.
28

29
# NOTE: If you perform changes to this script, also consider changing the template!
30

31
32
33
34
35
36
37
# This is a simple mirroring script. To save bandwidth it first checks a
# timestamp via HTTP and only runs rsync when the timestamp differs from the
# local copy. As of 2016, a single rsync run without changes transfers roughly
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
# is run every minute. Performing a simple check via HTTP first can thus save a
# lot of traffic.

38
# Directory where the repo is stored locally. Example: /srv/repo
39
target="/srv/ftp"
40
41
42
43

# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $target, but not a subdirectory of $target.
# Example: /srv/tmp
44
tmp="/srv/syncrepo-tmp"
45
46

# Lockfile path
47
48
lock="/var/lock/syncrepo.lck"

49
50
# If you want to limit the bandwidth used by rsync set this.
# Use 0 to disable the limit.
51
52
53
# The default unit is KiB (see man rsync /--bwlimit for more)
bwlimit=0

54
55
56
57
58
# The source URL of the mirror you want to sync from.
# If you are a tier 1 mirror use rsync.archlinux.org, for example like this:
# rsync://rsync.archlinux.org/ftp_tier1
# Otherwise chose a tier 1 mirror from this list and use its rsync URL:
# https://www.archlinux.org/mirrors/
59
source_url='rsync://rsync.archlinux.org/ftp_tier1'
60
61

# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
62
# If you are a tier 1 mirror use: https://rsync.archlinux.org/lastupdate
63
# Otherwise use the HTTP(S) URL from your chosen mirror.
64
lastupdate_url='https://rsync.archlinux.org/lastupdate'
65
66

#### END CONFIG
67
68

[ ! -d "${target}" ] && mkdir -p "${target}"
69
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"
70

71
72
73
# Cleanup any temporary files from old run that might remain.
find "${target}" -name '.~tmp~' -exec rm -r {} +

74
75
76
exec 9>"${lock}"
flock -n 9 || exit

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
rsync_cmd() {
	local -a cmd=(rsync -rtlH --safe-links --delete-after ${VERBOSE} "--timeout=600" "--contimeout=60" -p \
		--delay-updates --no-motd "--temp-dir=${tmp}")

	if stty &>/dev/null; then
		cmd+=(-h -v --progress)
	else
		cmd+=("--info=name1")
	fi

	if ((bwlimit>0)); then
		cmd+=("--bwlimit=$bwlimit")
	fi

	"${cmd[@]}" "$@"
}
93

94
95

# if we are called without a tty (cronjob) only run when there are changes
96
if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
97
	# keep lastsync file in sync for statistics generated by the Arch Linux website
98
	rsync_cmd "$source_url/lastsync" "$target/lastsync"
99
100
101
	exit 0
fi

102
rsync_cmd \
103
104
	--exclude='/other' \
	--exclude='/sources' \
105
	"${source_url}" \
106
107
108
	"${target}"

#echo "Last sync was $(date -d @$(cat ${target}/lastsync))"