Pablo Volenski
7f7f535bda
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
31 lines
1.1 KiB
Bash
31 lines
1.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Download and convert the OPUS-MT models into $DEST (default /srv/mt-models).
|
|
# Containers run as root so pip lands on PATH and can write to the root-owned
|
|
# target; the pipeline mounts that directory read-only.
|
|
set -euo pipefail
|
|
DEST="${1:-/srv/mt-models}"
|
|
MODELS=(
|
|
"Helsinki-NLP/opus-mt-es-de"
|
|
"Helsinki-NLP/opus-mt-tc-big-de-es"
|
|
"Helsinki-NLP/opus-mt-tc-big-itc-itc"
|
|
)
|
|
for REPO in "${MODELS[@]}"; do
|
|
NAME="${REPO##*/}"
|
|
if [ -f "$DEST/$NAME/model.bin" ]; then
|
|
echo "== $NAME already present, skipping"
|
|
continue
|
|
fi
|
|
echo "== converting $NAME"
|
|
docker run --rm --memory=3g -v "$DEST:/out" python:3.12-slim bash -c "
|
|
set -e
|
|
pip install --quiet 'transformers[torch]' ctranslate2 sentencepiece \
|
|
--extra-index-url https://download.pytorch.org/whl/cpu
|
|
ct2-transformers-converter --model $REPO --output_dir /out/$NAME --quantization int8
|
|
python -c 'import sys
|
|
from transformers import AutoTokenizer
|
|
AutoTokenizer.from_pretrained(sys.argv[1]).save_pretrained(sys.argv[2])' $REPO /out/$NAME
|
|
"
|
|
done
|
|
echo
|
|
du -sh "$DEST"/*
|