-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathimport.sh
executable file
·194 lines (160 loc) · 4.88 KB
/
import.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
#!/usr/bin/env bash
# Init the import scripts for all stores
# ------------------------
# - get stores from docker-compose environment vars STORE_[index] (see README)
# - if ssh is available:
# . copy import data, send store-specific import-script and execute
# - after import, start nginx as request ping service and hanging around idle
# ------------------------
store_import_dir='/import_store'
: ${CONNECTION_ATTEMPTS:=10}
# main function
main() {
if [ ! -d ${store_import_dir} ] || [ "$(ls -A ${store_import_dir} 2> /dev/null)" == "" ]; then
echo "[WARNING] import folder is empty. Will not load any data"
start_request_service
fi
cd "$store_import_dir"
bz2_to_gz "$store_import_dir"
# load and parse stores from env_stores
i=1
stores="STORE_${i}"
while [ -n "${!stores}" ]
do
store=${!stores}
# echo "[INFO] store $i config: '${store}'"
URI=""
HOST=""
PORT=""
TYPE=""
USER=""
PWD=""
get_store_config "$store"
echo "[INFO] got store $i (${URI})"
# echo "type: $TYPE , uri: $URI , user: $USER , pwd: $PWD"
if [[ -z "$URI" || -z "$HOST" ]] ; then
echo "[ERROR] empty host or uri of store $i. If your are using Docker Compose check the links and environment ids."
exit 1
fi
echo "[INFO] waiting for store $i to come online"
test_connection "${CONNECTION_ATTEMPTS}" "${HOST}" "${PORT}"
if [ $? -eq 2 ]; then
echo "[ERROR] store ${HOST}:${PORT} not reachable"
exit 1
else
echo "[INFO] store $i connection OK"
fi
/import-virtuoso.sh "${HOST}" "${PORT}" "${USER}" "${PWD}"
let "i=$i+1"
stores="STORE_${i}"
done
start_request_service
} # end of main
# test connection to host and port
test_connection () {
if [[ -z $1 || -z $2 ]]; then
echo "[ERROR] missing argument: retry attempts or host"
exit 1
fi
t=$1
host=$2
port=$3
if [[ -z $port ]]; then
echo "[WARNING] no port given for connection-test, set 80 as default."
port=80
fi
nc -w 1 "$host" $port < /dev/null;
#curl --output /dev/null --silent --head --fail "$host"
while [[ $? -ne 0 ]] ;
do
echo -n "..."
sleep 2
echo -n $t
let "t=$t-1"
if [ $t -eq 0 ]
then
echo "...timeout"
return 2
fi
nc -w 1 "$host" $port < /dev/null;
done
echo ""
}
bz2_to_gz () {
if [[ -z "$1" || ! -d "$1" ]]; then
echo "[ERROR] not a valid directory path: $wd"
exit 1
fi
wd="$1"
bz2_archives=( "$wd"/*bz2 )
bz2_archive_count=${#bz2_archives[@]}
if [[ $bz2_archive_count -eq 0 || ( $bz2_archive_count -eq 1 && "$bz2_archives" == "${wd}/*bz2" ) ]]; then
return 0
fi
echo "[INFO] converting $bz2_archive_count bzip2 archives to gzip:"
for archive in ${bz2_archives[@]}; do
echo "[INFO] converting $archive"
pbzip2 -dc $archive | pigz - > ${archive%bz2}gz
rm $archive
done
}
# get uri from linked docker container
uri_store_matching() {
uri=$1
# may get uri from %store_id%
#if [[ "$uri" =~ %[A-Za-z]+% ]] ; then
if [[ "$uri" =~ %%.*%% ]] ; then
match=$BASH_REMATCH # = %%store_id%%
appendix=${uri//*$match/}
prefix=${uri//$match*/}
store_id=${match//\%/}
store_tcp_var="${store_id^^}_PORT" # address variable with uppercased store_id
store_tcp=${!store_tcp_var} # store tcp address
store_tcp=${store_tcp//*\//} # remove tcp://
store_tcp=${store_tcp//:*/} # remove port
uri=${prefix}${store_tcp}${appendix}
#uri=${uri/$store_tcp/$match/}
fi
echo $uri
}
# get config (URI, TYPE, USER, PWD) from string
get_store_config () {
if [[ -z $1 ]]; then
echo "[ERROR] missing argument: store variable"
exit 1
fi
store=$1
for str in $store
do
# split string at delimiter '=>'
arr=(${str//=>/ })
key=${arr[0]}
val=${arr[1]}
# echo "${key} = ${val}"
case "$key" in
"uri" )
URI=$(uri_store_matching $val)
HOST=${URI//*:\/\//} # remove http://
HOST=${HOST//:*/} # remove port/path
HOST=${HOST//\/*/}
PORT=${URI//*:/}
PORT=${PORT//\/*/}
;;
"type" )
TYPE=$val
;;
"user" )
USER=$val
;;
"pwd" )
PWD=$val
;;
esac
done
}
start_request_service () {
echo "[INFO] Done. Hanging around idle and listen on port 80 for requests"
nc -kl 80
}
# start main
main