# Copyright 2015-present Lenovo
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Bootstrap:docker
From:nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04

%files
    llama-factory/llama-factory.conf /scripts/llama-factory.conf
    llama-factory/start /scripts/start
    /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem

%post
    rm -f /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/nvidia-ml.list
    apt-get update
    apt-get upgrade -y
    apt-get install -y git pip

    git clone --depth 1  https://github.com/lenovo/LLaMA-Factory.git /opt/LLaMA-Factory
    chmod -R 777 /opt/LLaMA-Factory
    cd /opt/LLaMA-Factory
    mkdir -p cache
    mkdir -p data
    mkdir -p saves
    mkdir -p config
    pip install -e ".[torch,metrics]"

    chmod -R 777 /scripts
    
    apt-get install -y nginx
    touch /run/nginx.pid
    chown www-data:www-data /run/nginx.pid
    chmod 644 /run/nginx.pid
    chown www-data:www-data /run
    chmod 755 /run
    chmod -R 777 /var/lib/nginx/
    chmod -R 777 /var/log/nginx/
    chmod -R 777 /etc/nginx/conf.d/
    sed -i "/include \/etc\/nginx\/modules-enabled\/\*\.conf;/d" /etc/nginx/nginx.conf
    sed -i "/include \/etc\/nginx\/sites-enabled\/\*;/d" /etc/nginx/nginx.conf

    # Install the requirements
    pip config set global.index-url "$PIP_INDEX"
    pip config set global.extra-index-url "$PIP_INDEX"
    python3 -m pip install --upgrade pip
    pip install -r requirements.txt

    # Install the LLaMA Factory
    EXTRA_PACKAGES="metrics"
    if [ "$INSTALL_BNB" == "true" ]; then
        EXTRA_PACKAGES="${EXTRA_PACKAGES},bitsandbytes"
    fi
    if [ "$INSTALL_VLLM" == "true" ]; then
        EXTRA_PACKAGES="${EXTRA_PACKAGES},vllm"
    fi
    if [ "$INSTALL_DEEPSPEED" == "true" ]; then
        EXTRA_PACKAGES="${EXTRA_PACKAGES},deepspeed"
    fi
    if [ "$INSTALL_LIGER_KERNEL" == "true" ]; then
        EXTRA_PACKAGES="${EXTRA_PACKAGES},liger-kernel"
    fi
    if [ "$INSTALL_HQQ" == "true" ]; then
        EXTRA_PACKAGES="${EXTRA_PACKAGES},hqq"
    fi
    if [ "$INSTALL_EETQ" == "true" ]; then
        EXTRA_PACKAGES="${EXTRA_PACKAGES},eetq"
    fi
    pip install -e ".[$EXTRA_PACKAGES]"

    # Rebuild flash attention if requested
    pip uninstall -y transformer-engine flash-attn
    if [ "$INSTALL_FLASHATTN" == "true" ]; then
        pip uninstall -y ninja
        pip install ninja
        pip install --no-cache-dir flash-attn --no-build-isolation
    fi
    
    # Set shared memory size for container
    mkdir -p /dev/shm
    mount -t tmpfs -o size=16G tmpfs /dev/shm

%environment
    export MAX_JOBS=4
    export FLASH_ATTENTION_FORCE_BUILD=TRUE
    export VLLM_WORKER_MULTIPROC_METHOD=spawn
    export GRADIO_SERVER_PORT=7860
    export API_PORT=8000
    export INSTALL_BNB=false
    export INSTALL_VLLM=false
    export INSTALL_DEEPSPEED=false
    export INSTALL_FLASHATTN=false
    export INSTALL_LIGER_KERNEL=false
    export INSTALL_HQQ=false
    export INSTALL_EETQ=false
    export PIP_INDEX=https://pypi.org/simple

%labels
    Maintainer "Liviu Muresanu <lmuresanu@lenovo.com>"
 llama-factory/llama-factory.conf deleted  100644 → 0
+
0
−
16

Viewed
server {
    listen LLAMA_FACTORY_NGINX_PORT;
    location ~ ^/dev/llama-factory/(?<address>[-\w]+)/(.*?)/(?<suffix>.*) {
        rewrite ^/dev/llama-factory/(.*?)/(.*?)/(.*?)$ /$3 break;
        proxy_pass http://127.0.0.1:GRADIO_SERVER_PORT;
        proxy_redirect http:// https://;
        client_max_body_size 5000m;
        proxy_intercept_errors on;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_set_header Host $http_host;
        proxy_set_header X-Forwarded-Host $http_host;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}