add ipex bench code

examples/benchmark/ipex.py

@@ -0,0 +1,71 @@
+import time
+import torch
+import argparse
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM
+from gptqmodel import GPTQModel, BACKEND
+
+def prepare_dataset_for_bench(tokenizer, batch_size=8):
+    dataset = load_dataset("json", data_files="prompts.jsonl", split="train")[:batch_size]
+    prompts = [[{"role": "user", "content": data}] for data in dataset['input']]
+    input_tensors = tokenizer.apply_chat_template(prompts, add_generation_prompt=True, return_tensors="pt", padding=True)
+    return input_tensors
+
+parser = argparse.ArgumentParser(description="Benchmark IPEX vs HF on a pre-trained model.")
+parser.add_argument("--model", type=str, required=True, help="Path or name of the pre-trained model.")
+parser.add_argument("--cores", type=int, default=8, help="Number of CPU cores to use.")
+parser.add_argument("--batch", type=int, default=8, help="Batch size for processing messages.")
+parser.add_argument("--backend", type=str, choices=["ipex", "hf"], default="ipex", help="Backend to optimize the model. Choose between 'ipex' and 'hf'.")
+ars = parser.parse_args()
+
+print("use model: ", ars.model)
+print("use cores: ", ars.cores)
+print("use batch: ", ars.batch)
+print("use backend: ", ars.backend)
+
+# Set the number of threads to use
+torch.set_num_threads(ars.cores)
+
+# load model, check model backend
+config = AutoConfig.from_pretrained(ars.model)
+is_quantized_model = hasattr(config, "quantization_config")
+if is_quantized_model:
+    print("load quantized model, will use BACKEND.IPEX")
+    model = GPTQModel.load(ars.model, backend=BACKEND.IPEX)
+else:
+    model = AutoModelForCausalLM.from_pretrained(ars.model, device_map="cpu", torch_dtype=torch.bfloat16)
+
+# set model to eval mode
+model.eval()
+
+if ars.backend == "ipex" and not is_quantized_model:
+    import intel_extension_for_pytorch as ipex
+    model = ipex.llm.optimize(model, dtype=torch.bfloat16)
+    model.forward = torch.compile(model.forward, dynamic=True, backend="ipex")
+
+# load tokenizer, and normalize the pad_token_id
+tokenizer = AutoTokenizer.from_pretrained(ars.model)
+tokenizer.pad_token_id = 128004 # <|finetune_right_pad_id|>
+model.pad_token_id = tokenizer.pad_token_id
+
+# prepare dataset for benchmark
+input_ids = prepare_dataset_for_bench(tokenizer, ars.batch)
+
+# benchmark
+with torch.no_grad(), torch.amp.autocast("cpu", dtype=torch.bfloat16):
+    start = time.time()
+    new_token_len = 0
+    outputs = model.generate(input_ids=input_ids.to(model.device), max_new_tokens=512, pad_token_id=tokenizer.pad_token_id)
+    for i, output in enumerate(outputs):
+        new_token_len += len(output) - len(input_ids[i])
+        # debug print
+        # result = tokenizer.decode(output, skip_special_tokens=False)
+        # print(result)
+        # print("="*50)
+
+    total_time = time.time() - start
+
+# display benchmark result
+print(f"generate use :{total_time}")
+print(f"total new token: {new_token_len}")
+print(f"token/sec: {(new_token_len/total_time):.4f}")

examples/benchmark/prompts.jsonl

@@ -0,0 +1,100 @@
+{"type":"Childcare","input":"What are the effects of parental involvement in children's education on their academic success?"}
+{"type":"WWII Weapons","input":"What were the specifications and operational roles of the Japanese Mitsubishi A6M Zero fighter aircraft?"}
+{"type":"Philosophy","input":"Discuss the principles of libertarianism in political philosophy and how they contrast with other political ideologies."}
+{"type":"Dinosaur","input":"What role did competition and predation play in the diversification of dinosaur species?"}
+{"type":"Home Economics","input":"What are the key principles of interior design?"}
+{"type":"Real Estate","input":"How do urban planning policies and infrastructure development projects influence long-term real estate trends in a city?"}
+{"type":"Game","input":"Can you explain the process of porting a game from one platform to another, including the technical and business considerations involved?"}
+{"type":"Eintsten Physcis","input":"What experimental evidence supports the theory of general relativity?"}
+{"type":"Home Economics","input":"How to choose and care for different types of fabrics?"}
+{"type":"Dentistry","input":"What precautions should be taken with dental crowns?"}
+{"type":"Economics","input":"Discuss how price elasticity affects consumer demand and business pricing strategies."}
+{"type":"Entertainment Stars","input":"Explain the journey of Dwayne \"The Rock\" Johnson from wrestling to acting."}
+{"type":"Excel","input":"How do you use the OFFSET function in Excel to create dynamic ranges?  "}
+{"type":"Economics","input":"How do tariffs and quotas influence international trade and domestic industries?"}
+{"type":"Famous People","input":"What role did Alan Turing play in the development of computer science?"}
+{"type":"HRM","input":"What are the key components of a successful employee recognition program?  "}
+{"type":"Memory and Counting","input":"Without looking, what is the 10th letter of the English alphabet?"}
+{"type":"Eintsten Physcis","input":"Explain the difference between Einstein's special relativity and general relativity."}
+{"type":"Philosophy","input":"Compare deontological ethics with consequentialist ethics and discuss their applications in moral decision-making."}
+{"type":"Capybara","input":"What differences exist between male and female capybaras in terms of size and behavior?"}
+{"type":"Excel","input":"How does one use Excel's advanced filter to extract specific data from a large dataset?"}
+{"type":"Famous People","input":"What was the significance of René Descartes' philosophical statement, \"I think, therefore I am\"?  "}
+{"type":"Dinosaur","input":"How have digital technologies, like 3D modeling and scanning, advanced dinosaur paleontology?"}
+{"type":"Eintsten Physcis","input":"How do Einstein's theories of relativity influence our understanding of cosmological phenomena like the Big Bang and the expansion of the universe?"}
+{"type":"World","input":"How did Mexico achieve its independence?"}
+{"type":"Traffic Law","input":"How are traffic laws enforced through the use of speed cameras and automatic ticketing in the Netherlands?"}
+{"type":"World","input":"How has the Aztec civilization influenced Mexican culture?"}
+{"type":"Infamous People","input":"Explain the criminal empire of Dawood Ibrahim and his influence on global organized crime."}
+{"type":"Real Estate","input":"What are the primary methods of real estate valuation, and when is each method most appropriately used?"}
+{"type":"Dinosaur","input":"How did the teeth and jaw structures of different dinosaurs reflect their diets and feeding habits?"}
+{"type":"Dentistry","input":"What are the causes of chronic dry mouth, and how can it be managed?  "}
+{"type":"Musical Instruments","input":"Discuss the role of timpani in orchestras and how they contribute to rhythm and harmony.  "}
+{"type":"Modern Theoretical Physics","input":"What are emergent phenomena in condensed matter physics, and how do they provide insights into complex quantum systems?"}
+{"type":"Entertainment Stars","input":"How do celebrities maintain their public image?"}
+{"type":"Game","input":"Describe the role of game designers in the development process."}
+{"type":"Common Knowledge","input":"What are the benefits of learning a second language, both personally and professionally?"}
+{"type":"Modern Theoretical Physics","input":"How do extra dimensions in theoretical physics models like Kaluza-Klein theory contribute to our understanding of fundamental forces?"}
+{"type":"Politics","input":"Explain how elections work in a democratic society."}
+{"type":"WWII Weapons","input":"Discuss the design and effectiveness of the German Panzerfaust anti-tank weapon in infantry engagements."}
+{"type":"Dentistry","input":"How does poor oral hygiene contribute to systemic diseases like heart disease and diabetes?  "}
+{"type":"Virology","input":"How do RNA interference mechanisms in host cells act as antiviral defenses, and how might viruses counteract these mechanisms?"}
+{"type":"World","input":"What were the purposes behind the construction of the ancient Egyptian pyramids?"}
+{"type":"Spanish","input":"Please translate \"Ayer fui al mercado y compré muchas frutas frescas.\" from Spanish to English."}
+{"type":"Journalism","input":"How do news organizations handle breaking news situations?"}
+{"type":"Geography","input":"What are the primary causes of desertification?"}
+{"type":"World","input":"What impact did the Greek debt crisis have on the Eurozone economy?"}
+{"type":"Eintsten Physcis","input":"Describe the concept of time dilation and provide an example."}
+{"type":"Criminology","input":"What is the role of forensic psychology in criminal investigations?"}
+{"type":"Aerospace","input":"Analyze the potential impact of emerging technologies such as artificial intelligence and quantum computing on aerospace engineering and space exploration."}
+{"type":"Aerospace","input":"Explain the function of gyroscopes and accelerometers in aircraft navigation systems.  "}
+{"type":"Geography","input":"Describe the process of coastal erosion."}
+{"type":"Game","input":"Discuss the impact of virtual reality on modern gaming."}
+{"type":"World","input":"How did the Spanish language become a global language?"}
+{"type":"Stock Market","input":"How has the proliferation of commission-free trading apps affected stock market participation and price volatility?"}
+{"type":"WWII Weapons","input":"What were the differences in design, performance, and roles between the German Bf 109 and Fw 190 fighter aircraft?"}
+{"type":"Real Estate","input":"What are the different real estate brokerage models, and how do they impact agents and clients?"}
+{"type":"Journalism","input":"What is the role of public relations in shaping news narratives?  "}
+{"type":"Geography","input":"Discuss the impacts of climate change on polar ice caps and the resulting effects on global sea levels and weather patterns."}
+{"type":"Graphic Design","input":"How does proportion and scale impact the effectiveness of a design?  "}
+{"type":"Newton Physics","input":"What is impulse, and how does it relate to the change in momentum of an object?"}
+{"type":"Aerospace","input":"What are the advantages and limitations of vertical takeoff and landing (VTOL) aircraft?  "}
+{"type":"Nutrition","input":"Is intermittent fasting effective for fat loss?"}
+{"type":"Puzzels","input":"A man was born in 1995 and died in 1953 at the age of 25. How is this possible?"}
+{"type":"Consumption","input":"How do environmental concerns shape the growing demand for eco-friendly products among consumers?  "}
+{"type":"Animal Husbandry","input":"Best practices for milking goats?"}
+{"type":"Animal Husbandry","input":"How do you train a working dog for herding?"}
+{"type":"Geology","input":"How does weathering contribute to soil formation and landscape development?"}
+{"type":"Puzzels","input":"What can run but never walks, has a mouth but never talks, has a head but never weeps, has a bed but never sleeps?"}
+{"type":"Journalism","input":"What is the role of a news editor in managing the flow of news within a newsroom?  "}
+{"type":"Religion","input":"What is the significance of religious festivals in fostering community and faith in different religions?  "}
+{"type":"Musical Instruments","input":"What is the significance of the viola in string quartets and orchestral settings?  "}
+{"type":"Geology","input":"What is the structure of Earth's interior?"}
+{"type":"Biology","input":"Describe the structure of DNA and its role in heredity."}
+{"type":"Eintsten Physcis","input":"What is frame-dragging in general relativity, and how has it been observed experimentally?"}
+{"type":"Preschool Education","input":"What are the benefits of using positive reinforcement in managing preschoolers' behavior, and how can it be effectively implemented?"}
+{"type":"Geology","input":"How do geologists identify and classify different types of sedimentary structures?"}
+{"type":"Traffic Law","input":"What are the regulations for child safety seats in vehicles according to French traffic law?"}
+{"type":"Modern Theoretical Physics","input":"What are the leading theories regarding the nature and composition of dark matter, and what experimental approaches are used to detect it?"}
+{"type":"Drugs","input":"What are the benefits and potential side effects of clopidogrel?"}
+{"type":"Tea Culture","input":"Describe the cultural importance of tea in the Georgian Supra feast.  "}
+{"type":"Biology","input":"Describe the process of PCR (Polymerase Chain Reaction) and its applications."}
+{"type":"Eintsten Physcis","input":"How does general relativity account for the perihelion advance of planetary orbits beyond Mercury?"}
+{"type":"Real Estate","input":"How do homeowners associations (HOAs) operate, and what are their typical responsibilities?"}
+{"type":"Psychology","input":"What are the main symptoms of depression?"}
+{"type":"Religion","input":"How do creation myths vary across religions?"}
+{"type":"Psychology","input":"What are the long-term psychological effects of bullying on victims?  "}
+{"type":"Graphic Design","input":"How do texture and patterns contribute to the overall feel of a graphic design?  "}
+{"type":"Criminology","input":"Define recidivism in criminology."}
+{"type":"World","input":"How does agriculture affect Argentina's economy?"}
+{"type":"Art and Culture","input":"Name a traditional dance from Greece."}
+{"type":"Biology","input":"Define biodiversity."}
+{"type":"Middle East Situation","input":"How do historical grievances and colonial legacies continue to shape current conflicts in the Middle East?"}
+{"type":"Common Knowledge","input":"What are some effective time management techniques that can improve productivity in daily life?"}
+{"type":"World","input":"How was German reunification achieved, and what were the main challenges faced?"}
+{"type":"Common Knowledge","input":"Describe the process of booking a flight online, including how to compare prices and select seats."}
+{"type":"Excel","input":"How can macros be used to automate repetitive tasks in Excel?"}
+{"type":"Traffic Law","input":"Describe the process for appealing a traffic ticket in the United Kingdom."}
+{"type":"World History","input":"Analyze the impact of the Enlightenment on the American Revolution."}
+{"type":"Eintsten Physcis","input":"What are the theoretical implications of merging black holes for our understanding of spacetime?"}
+{"type":"Stock Market","input":"What is market liquidity, and why is it important for the efficient functioning of stock markets?"}