Backport packages: support paginated list of artifacts for v5 correctly (#899)

Signed-off-by: Kentaro Hayashi <[email protected]>

Author: kenhys

fluent-package/manage-fluent-repositories.sh

@@ -197,78 +197,79 @@ EOF
 	     https://api.github.com/repos/fluent/fluent-package-builder/pulls/$PULL_NUMBER | jq --raw-output '.head | .ref + " " + .sha')
 	head_branch=$(echo $response | cut -d' ' -f1)
 	head_sha=$(echo $response | cut -d' ' -f2)
-	rm -f dl.list
-	for d in `seq 1 5`; do
+	rm -f dl.*.list
+	for d in `seq 1 20`; do
 	    curl --silent --location \
 		 -H "Accept: application/vnd.github+json" \
 		 -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" \
 		 -H "X-GitHub-Api-Version: 2022-11-28" \
 		 "https://api.github.com/repos/fluent/fluent-package-builder/actions/artifacts?per_page=100&page=$d" | \
-		jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + (.size_in_bytes|tostring) + " " + .archive_download_url' > dl.list
-	    filesize=$(stat -c "%s" dl.list)
+		jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + (.size_in_bytes|tostring) + " " + .archive_download_url' > dl.$d.list
+	    filesize=$(stat -c "%s" dl.$d.list)
+
 	    if [ $filesize -eq 0 ]; then
 		echo "Failed to fetch artifacts list"
-		rm -f dl.list
+		rm -f dl.$d.list
 		continue
-	    else
-		break
 	    fi
-	done
-	while read line
-	do
-	    package=$(echo $line | cut -d' ' -f1)
-	    download_url=$(echo $line | cut -d' ' -f3)
-	    echo "Downloading $package.zip from $download_url"
-	    case $package in
-		*debian*|*ubuntu*)
-		    mkdir -p apt/repositories
-		    (cd apt/repositories &&
-			 rm -f $package.zip &&
-			 curl --silent --location --output $package.zip \
-			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
-		    ;;
-		*centos*|*rockylinux*|*almalinux*|*amazonlinux*)
-		    mkdir -p yum/repositories
-		    (cd yum/repositories &&
-			 rm -f $package.zip &&
-			 curl --silent --location --output $package.zip \
-			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
-		    ;;
-		*)
-		    curl --silent --location --output $package.zip \
-			 -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &
-		    ;;
-	    esac
-	done < dl.list
-	wait
+	    while read line
+	    do
+		package=$(echo $line | cut -d' ' -f1)
+		download_url=$(echo $line | cut -d' ' -f3)
+		echo "Downloading $package.zip from $download_url"
+		case $package in
+		    *debian*|*ubuntu*)
+			mkdir -p apt/repositories
+			(cd apt/repositories &&
+			     rm -f $package.zip &&
+			     curl --silent --location --output $package.zip \
+				  -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
+			;;
+		    *centos*|*rockylinux*|*almalinux*|*amazonlinux*)
+			mkdir -p yum/repositories
+			(cd yum/repositories &&
+			     rm -f $package.zip &&
+			     curl --silent --location --output $package.zip \
+				  -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
+			;;
+		    *)
+			curl --silent --location --output $package.zip \
+			     -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &
+			;;
+		esac
+	    done < dl.$d.list
+	    wait
 
-	verified=1
-	while read line
-	do
-	    package=$(echo $line | cut -d' ' -f1)
-	    download_size=$(echo $line | cut -d' ' -f2)
-	    case $package in
-		*debian*|*ubuntu*)
-		    actual_size=$(stat --format="%s" apt/repositories/$package.zip)
-		    ;;
-		*rockylinux*|*almalinux*|*amazonlinux*)
-		    actual_size=$(stat --format="%s" yum/repositories/$package.zip)
-		    ;;
-		*)
-		    actual_size=$(stat --format="%s" $package.zip)
-		    ;;
-	    esac
-	    if [ $download_size = "$actual_size" ]; then
-		echo -e "[\e[32m\e[40mOK\e[0m] Verify apt/repositories/$package.zip"
-	    else
-		echo -e "[\e[31m\e[40mNG\e[0m] Verify apt/repositories/$package.zip (expected: $download_size actual: $actual_size)"
-		verified=0
+	    verified=1
+	    while read line
+	    do
+		package=$(echo $line | cut -d' ' -f1)
+		download_size=$(echo $line | cut -d' ' -f2)
+		case $package in
+		    *debian*|*ubuntu*)
+			actual_size=$(stat --format="%s" apt/repositories/$package.zip)
+			;;
+		    *rockylinux*|*almalinux*|*amazonlinux*)
+			actual_size=$(stat --format="%s" yum/repositories/$package.zip)
+			;;
+		    *)
+			actual_size=$(stat --format="%s" $package.zip)
+			;;
+		esac
+		if [ $download_size = "$actual_size" ]; then
+		    echo -e "[\e[32m\e[40mOK\e[0m] Verify apt/repositories/$package.zip"
+		else
+		    echo -e "[\e[31m\e[40mNG\e[0m] Verify apt/repositories/$package.zip (expected: $download_size actual: $actual_size)"
+		    verified=0
+		fi
+	    done < dl.$d.list
+	    if [ $verified -eq 0 ]; then
+		echo "Downloaded artifacts were corrupted! Check the downloaded artifacts."
+		exit 1
 	    fi
-	done < dl.list
-	if [ $verified -eq 0 ]; then
-	    echo "Downloaded artifacts were corrupted! Check the downloaded artifacts."
-	    exit 1
-	fi
+	done
+	mkdir -p apt/repositories
+	mkdir -p yum/repositories
 	(cd apt/repositories && find . -name '*.zip' -exec unzip -u -o {} \;)
 	(cd yum/repositories && find . -name '*.zip' -exec unzip -u -o {} \;)
 	;;