From 634d9341ca6d6723fbc46208f3c461098cdfa550 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Nov 2023 18:03:52 +0800 Subject: [PATCH 1/2] Support disable distrubted launch --- llm/fastdeploy_llm/utils/launch_infer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llm/fastdeploy_llm/utils/launch_infer.py b/llm/fastdeploy_llm/utils/launch_infer.py index eca1198202..771c23332e 100644 --- a/llm/fastdeploy_llm/utils/launch_infer.py +++ b/llm/fastdeploy_llm/utils/launch_infer.py @@ -43,6 +43,19 @@ def launch(device_ids, **kwargs: dict): pd_cmd = "python3 -m paddle.distributed.launch --devices {} {} {}".format( device_ids, infer_script_path, ' '.join(args)) + + multicard = True + try: + ids = eval(device_ids) + if isinstance(ids, int): + multicard = False + except: + pass + + if os.getenv("DISABLE_DISTRIBUTED_LAUNCH", "OFF") == ON and not multicard: + logger.info("Since DISABLE_DISTRIBUTED_LAUNCH=ON and the model is running with single card, will disable paddle.distributed.launch.") + pd_cmd = "python3 {} {}".format(infer_script_path, ' '.join(args)) + logger.info("Launch model with command: {}".format(pd_cmd)) logger.info("Model is initializing...") p = subprocess.Popen( From 95c20c4b916cb5abdd79ae8f55dfa016056c0c1a Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Nov 2023 18:06:15 +0800 Subject: [PATCH 2/2] Support disable distrubted launch --- llm/fastdeploy_llm/utils/launch_infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/fastdeploy_llm/utils/launch_infer.py b/llm/fastdeploy_llm/utils/launch_infer.py index 771c23332e..526da36b98 100644 --- a/llm/fastdeploy_llm/utils/launch_infer.py +++ b/llm/fastdeploy_llm/utils/launch_infer.py @@ -52,7 +52,7 @@ def launch(device_ids, **kwargs: dict): except: pass - if os.getenv("DISABLE_DISTRIBUTED_LAUNCH", "OFF") == ON and not multicard: + if os.getenv("DISABLE_DISTRIBUTED_LAUNCH", "OFF") == "ON" and not multicard: logger.info("Since DISABLE_DISTRIBUTED_LAUNCH=ON and the model is running with single card, will disable paddle.distributed.launch.") pd_cmd = "python3 {} {}".format(infer_script_path, ' '.join(args))