Skip to content

Commit

Permalink
Use tmp dir in cwd for stats generation
Browse files Browse the repository at this point in the history
The /tmp partition on the new al2023 instances isn't big enough to
support stats file processing, so this moves it to the root partition
which has more space.

We should do something better longer term, but this should allow the
stat generation to at least complete.
  • Loading branch information
tobias committed Mar 6, 2024
1 parent 0b131eb commit 9a6b64d
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 12 deletions.
13 changes: 9 additions & 4 deletions scripts/build-search-index
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,16 @@ cd "$dir/.."

clojars_jar=$1

java -cp "$clojars_jar" clojure.main -m clojars.tools.build-search-index production | tee /tmp/index-out.txt
mkdir -p tmp

index_path=$(grep "index-path:" /tmp/index-out.txt | sed -n 's/index-path: \(.*\)/\1/p')
index_out=tmp/index-out.txt
java -cp "$clojars_jar" clojure.main -m clojars.tools.build-search-index production | tee "$index_out"

index_path=$(grep "index-path:" "$index_out" | sed -n 's/index-path: \(.*\)/\1/p')

index_archive=tmp/artifact-index.tgz

# upload the new index the s3 bucket
tar czf /tmp/artifact-index.tgz "$index_path"
tar czf "$index_archive" "$index_path"

aws s3 cp --no-progress --content-type 'application/gzip' /tmp/artifact-index.tgz "s3://clojars-artifact-index/"
aws s3 cp --no-progress --content-type 'application/gzip' "$index_archive" "s3://clojars-artifact-index/"
4 changes: 3 additions & 1 deletion scripts/combine-cdn-logs
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ if [ -z "$DATE" ]; then
DATE=$(date --date='1 day ago' +%Y%m%d)
fi

mkdir -p tmp

S3_LOG_BUCKET=clojars-fastly-logs

java -cp "$clojars_jar" clojure.main -m clojars.tools.combine-cdn-logs \
"$S3_LOG_BUCKET" "$DATE" "$output_file"

if [ -s "$output_file" ]; then
gzipped_file="/tmp/combined-${DATE}.log.gz"
gzipped_file="tmp/combined-${DATE}.log.gz"
gzip --best --stdout "$output_file" > "$gzipped_file"
aws s3 cp --no-progress --content-type 'application/gzip' "$gzipped_file" "s3://${S3_LOG_BUCKET}/"
fi
4 changes: 3 additions & 1 deletion scripts/update-pom-list
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,6 @@ set -e

clojars_jar=$1

java -cp $clojars_jar clojure.main -m clojars.tools.generate-feeds production 2> /dev/null
mkdir -p tmp

java -cp $clojars_jar clojure.main -m clojars.tools.generate-feeds tmp production 2> /dev/null
9 changes: 5 additions & 4 deletions scripts/update-stats
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,17 @@ if [ -z "$date" ]; then
date=$(date --date='1 day ago' +%Y%m%d)
fi

cdn_logfile=/tmp/cdn-access-yesterday.log
mkdir -p tmp
cdn_logfile=tmp/cdn-access-yesterday.log

# generate the combined cdn stats first
"$dir"/combine-cdn-logs "$clojars_jar" "$cdn_logfile" "$date"

S3_STATS_BUCKET=clojars-stats-production

downloads_date=/tmp/downloads-$date.edn
downloads_all_old=/tmp/all-old.edn
downloads_all_new=/tmp/all-new.edn
downloads_date=tmp/downloads-$date.edn
downloads_all_old=tmp/all-old.edn
downloads_all_new=tmp/all-new.edn

# grab the latest all.edn from s3
aws s3 cp "s3://${S3_STATS_BUCKET}/all.edn" "$downloads_all_old"
Expand Down
4 changes: 2 additions & 2 deletions src/clojars/tools/generate_feeds.clj
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@
(write-sums jar-file)
(write-sums gz-file)))))

(defn -main [env]
(defn -main [feed-dir env]
(let [{:keys [db s3]} (config (keyword env))]
(generate-feeds "/tmp"
(generate-feeds feed-dir
db
(s3/s3-client (:repo-bucket s3)))))

0 comments on commit 9a6b64d

Please sign in to comment.