@article{duan2024LLMSurvey,title={Efficient Training of Large Language Models on Distributed Infrastructures: A Survey},author={Duan, Jiangfei and Zhang, Shuo and Wang, Zerui and Jiang, Lijuan and Qu, Wenwen and Hu, Qinghao and Wang, Guoteng and Weng, Qizhen and Yan, Hang and Zhang, Xingcheng and Qiu, Xipeng and Lin, Dahua and Wen, Yonggang and Jin, Xin and Zhang, Tianwei and Sun, Peng},journal={arXiv preprint arXiv:2407.20018},publisher={arXiv},year={2024},}
InternLM2 Technical Report
Zheng Cai, Maosong Cao, Haojiong Chen, Kai Chen, Keyu Chen, Xin Chen, Xun Chen, Zehui Chen, Zhi Chen, Pei Chu, and
90 more authors
@article{cai2024Internlm2,title={InternLM2 Technical Report},author={Cai, Zheng and Cao, Maosong and Chen, Haojiong and Chen, Kai and Chen, Keyu and Chen, Xin and Chen, Xun and Chen, Zehui and Chen, Zhi and Chu, Pei and Dong, Xiaoyi and Duan, Haodong and Fan, Qi and Fei, Zhaoye and Gao, Yang and Ge, Jiaye and Gu, Chenya and Gu, Yuzhe and Gui, Tao and Guo, Aijia and Guo, Qipeng and He, Conghui and Hu, Yingfan and Huang, Ting and Jiang, Tao and Jiao, Penglong and Jin, Zhenjiang and Lei, Zhikai and Li, Jiaxing and Li, Jingwen and Li, Linyang and Li, Shuaibin and Li, Wei and Li, Yining and Liu, Hongwei and Liu, Jiangning and Hong, Jiawei and Liu, Kaiwen and Liu, Kuikun and Liu, Xiaoran and Lv, Chengqi and Lv, Haijun and Lv, Kai and Ma, Li and Ma, Runyuan and Ma, Zerun and Ning, Wenchang and Ouyang, Linke and Qiu, Jiantao and Qu, Yuan and Shang, Fukai and Shao, Yunfan and Song, Demin and Song, Zifan and Sui, Zhihao and Sun, Peng and Sun, Yu and Tang, Huanze and Wang, Bin and Wang, Guoteng and Wang, Jiaqi and Wang, Jiayu and Wang, Rui and Wang, Yudong and Wang, Ziyi and Wei, Xingjian and Weng, Qizhen and Wu, Fan and Xiong, Yingtong and Xu, Chao and Xu, Ruiliang and Yan, Hang and Yan, Yirong and Yang, Xiaogui and Ye, Haochen and Ying, Huaiyuan and Yu, Jia and Yu, Jing and Zang, Yuhang and Zhang, Chuyu and Zhang, Li and Zhang, Pan and Zhang, Peng and Zhang, Ruijie and Zhang, Shuo and Zhang, Songyang and Zhang, Wenjian and Zhang, Wenwei and Zhang, Xingcheng and Zhang, Xinyue and Zhao, Hui and Zhao, Qian and Zhao, Xiaomeng and Zhou, Fengzhe and Zhou, Zaida and Zhuo, Jingming and Zou, Yicheng and Qiu, Xipeng and Qiao, Yu and Lin, Dahua},journal={arXiv preprint arXiv:2403.17297},publisher={arXiv},year={2024},}
CaraServe: CPU-Assisted and Rank-Aware LoRA Serving for Generative LLM Inference
Suyi Li, Hanfeng Lu, Tianyuan Wu, Minchen Yu, Qizhen Weng, Xusheng Chen, Yizhou Shan, Binhang Yuan, and Wei Wang
@article{li2024CaraServe,title={CaraServe: CPU-Assisted and Rank-Aware {LoRA} Serving for Generative {LLM} Inference},author={Li, Suyi and Lu, Hanfeng and Wu, Tianyuan and Yu, Minchen and Weng, Qizhen and Chen, Xusheng and Shan, Yizhou and Yuan, Binhang and Wang, Wei},journal={arXiv preprint arXiv:2401.11240},publisher={arXiv},year={2024},}
2023
Beware of Fragmentation: Scheduling GPU-Sharing Workloads with Fragmentation Gradient Descent
@inproceedings{weng2023FGD,title={Beware of Fragmentation: Scheduling {GPU}-Sharing Workloads with Fragmentation Gradient Descent},author={Weng, Qizhen and Yang, Lingyun and Yu, Yinghao and Wang, Wei and Tang, Xiaochuan and Yang, Guodong and Zhang, Liping},booktitle={2023 {USENIX} Annual Technical Conference (ATC)},publisher={{USENIX} Association},year={2023},isbn={978-1-939133-35-9},address={Boston, MA},pages={995--1008},url={https://www.usenix.org/conference/atc23/presentation/weng},}
2022
MLaaS in the Wild: Workload Analysis and Scheduling in Large-Scale Heterogeneous GPU Clusters
@inproceedings{weng2022MLaaS,title={{MLaaS} in the Wild: Workload Analysis and Scheduling in Large-Scale Heterogeneous {GPU} Clusters},author={Weng, Qizhen and Xiao, Wencong and Yu, Yinghao and Wang, Wei and Wang, Cheng and He, Jian and Li, Yong and Zhang, Liping and Lin, Wei and Ding, Yu},booktitle={19th USENIX Symposium on Networked Systems Design and Implementation (NSDI)},pages={945--960},year={2022},}
Workload consolidation in Alibaba clusters: the good, the bad, and the ugly
Yongkang Zhang, Yinghao Yu, Wei Wang, Qiukai Chen, Jie Wu, Zuowei Zhang, Jiang Zhong, Tianchen Ding, Qizhen Weng, Lingyun Yang, and
4 more authors
In 13th ACM Symposium on Cloud Computing (SoCC), 2022
@inproceedings{zhang2022Workload,title={Workload consolidation in {Alibaba} clusters: the good, the bad, and the ugly},author={Zhang, Yongkang and Yu, Yinghao and Wang, Wei and Chen, Qiukai and Wu, Jie and Zhang, Zuowei and Zhong, Jiang and Ding, Tianchen and Weng, Qizhen and Yang, Lingyun and Wang, Cheng and He, Jian and Yang, Guodong and Zhang, Liping},booktitle={13th ACM Symposium on Cloud Computing (SoCC)},pages={210--225},year={2022},}
2021
Accelerating Distributed Learning in Non-Dedicated Environments
Chen Chen, Qizhen Weng, Wei Wang, Baochun Li, and Bo Li
@article{chen2021Accelerating,title={Accelerating Distributed Learning in Non-Dedicated Environments},author={Chen, Chen and Weng, Qizhen and Wang, Wei and Li, Baochun and Li, Bo},journal={IEEE Transactions on Cloud Computing (TCC)},year={2021},publisher={IEEE},}
2020
Metis: Learning to Schedule Long-Running Applications in Shared Container Clusters at Scale
Luping Wang, Qizhen Weng, Wei Wang, Chen Chen, and Bo Li
In International Conference for High Performance Computing, Networking, Storage and Analysis (SC), 2020
@inproceedings{wang2020Metis,title={Metis: Learning to Schedule Long-Running Applications in Shared Container Clusters at Scale},author={Wang, Luping and Weng, Qizhen and Wang, Wei and Chen, Chen and Li, Bo},booktitle={International Conference for High Performance Computing, Networking, Storage and Analysis (SC)},pages={1--17},year={2020},organization={IEEE},}
Semi-Dynamic Load Balancing: Efficient Distributed Learning in Non-Dedicated Environments
Chen Chen, Qizhen Weng, Wei Wang, Baochun Li, and Bo Li
In 11th ACM Symposium on Cloud Computing (SoCC), 2020
@inproceedings{chen2020Semi,title={Semi-Dynamic Load Balancing: Efficient Distributed Learning in Non-Dedicated Environments},author={Chen, Chen and Weng, Qizhen and Wang, Wei and Li, Baochun and Li, Bo},booktitle={11th ACM Symposium on Cloud Computing (SoCC)},pages={431--446},year={2020},}
2019
APSys
Towards Framework-Independent, Non-Intrusive Performance Characterization for Dataflow Computation
Huangshi Tian, Qizhen Weng, and Wei Wang
In Proceedings of the 10th ACM SIGOPS Asia-Pacific Workshop on Systems (APSys), 2019
@inproceedings{tian2019Towards,title={Towards Framework-Independent, Non-Intrusive Performance Characterization for Dataflow Computation},author={Tian, Huangshi and Weng, Qizhen and Wang, Wei},booktitle={Proceedings of the 10th ACM SIGOPS Asia-Pacific Workshop on Systems (APSys)},pages={54--60},year={2019},}
2018
SoCC
Fast Distributed Deep Learning via Worker-Adaptive Batch Sizing
Chen Chen, Qizhen Weng, Wei Wang, Baochun Li, and Bo Li
In 9th ACM Symposium on Cloud Computing (SoCC), 2018
@inproceedings{chen2018Fast,title={Fast Distributed Deep Learning via Worker-Adaptive Batch Sizing},author={Chen, Chen and Weng, Qizhen and Wang, Wei and Li, Baochun and Li, Bo},booktitle={9th ACM Symposium on Cloud Computing (SoCC)},pages={521--521},year={2018},}
ICDCS
Opus: Fair and Efficient Cache Sharing for In-Memory Data Analytics
Yinghao Yu, Wei Wang, Jun Zhang, Qizhen Weng, and Khaled Ben Letaief
In 38th IEEE International Conference on Distributed Computing Systems (ICDCS), 2018
@inproceedings{yu2018Opus,title={Opus: Fair and Efficient Cache Sharing for In-Memory Data Analytics},author={Yu, Yinghao and Wang, Wei and Zhang, Jun and Weng, Qizhen and Letaief, Khaled Ben},booktitle={38th IEEE International Conference on Distributed Computing Systems (ICDCS)},pages={154--164},year={2018},organization={IEEE},}
