fix transformers

Signed-off-by: Sun, Xuehao <[email protected]>

.github/workflows/scripts/freeze_requirements.sh

@@ -15,13 +15,17 @@ function freeze() {
         --output-file "$folder/freeze.txt" \
         "$file"
     echo "::endgroup::"
+
     if [[ -e "$folder/freeze.txt" ]]; then
         if [[ "$keep_origin_packages" == "true" ]]; then
+            # fix corner cases
             sed -i '/^\s*#/d; s/#.*//; /^\s*$/d; s/ //g' "$file"
             sed -i '/^\s*#/d; s/#.*//; /^\s*$/d; s/ //g; s/huggingface-hub\[inference\]/huggingface-hub/g; s/uvicorn\[standard\]/uvicorn/g' "$folder/freeze.txt"
-
+            if grep -q '^transformers$' $file && ! grep -q '^transformers\[sentencepiece\]$' $file; then
+                sed -i "s/transformers\[sentencepiece\]/transformers/" "$folder/freeze.txt"
+            fi
             packages1=$(tr '><' '=' <"$file" | cut -d'=' -f1 | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g')
-            packages2=$(cut -d'=' -f1 "$folder/freeze.txt" | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g' )
+            packages2=$(cut -d'=' -f1 "$folder/freeze.txt" | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g')
             common_packages=$(comm -12 <(echo "$packages2" | sort) <(echo "$packages1" | sort))
             grep '^git\+' "$file" >temp_file || touch temp_file
             rm -rf "$file" && mv temp_file "$file"
@@ -51,6 +55,7 @@ function check_branch_name() {
 
 function main() {
     check_branch_name
+    upgrade_transformers
     echo "::group::pip install pip-tools" && pip install pip-tools --upgrade && echo "::endgroup::"
     export -f freeze
     find . -name "requirements.txt" | xargs -n 1 -I {} bash -c 'freeze "$@"' _ {}