Update README.md

Add batch inference example.

README.md

@@ -78,6 +78,58 @@ with torch.inference_mode():
     print(f'Output:\n{output}')
 ```
 
+<details>
+<summary>Batch inference</summary>
+
+```python
+batch_inputs = [
+    ('example_image1.jpeg', 'Describe the content of this image.'),
+    ('example_image2.jpeg', 'What is the equation in the image?')
+]
+
+batch_input_ids = []
+batch_attention_mask = []
+batch_pixel_values = []
+
+for image_path, text in batch_inputs:
+    image = Image.open(image_path)
+    query = f'<image>\n{text}'
+    prompt, input_ids, pixel_values = model.preprocess_inputs(query, [image])
+    attention_mask = torch.ne(input_ids, text_tokenizer.pad_token_id)
+    input_ids = input_ids.unsqueeze(0).to(device=model.device)
+    attention_mask = attention_mask.unsqueeze(0).to(device=model.device)
+    pixel_values = [pixel_values.to(dtype=visual_tokenizer.dtype, device=visual_tokenizer.device)]
+    batch_input_ids.append(input_ids.squeeze())
+    batch_attention_mask.append(attention_mask.squeeze())
+    batch_pixel_values.append(pixel_values)
+
+pad_batch_input_ids = torch.nn.utils.rnn.pad_sequence([i.flip(dims=[0]) for i in batch_input_ids],batch_first=True, padding_value=0.0).flip(dims=[1])
+pad_batch_input_ids =  pad_batch_input_ids[:,-model.config.multimodal_max_length:]
+pad_batch_attention_mask = torch.nn.utils.rnn.pad_sequence([i.flip(dims=[0]) for i in batch_attention_mask],batch_first=True, padding_value=False).flip(dims=[1])
+pad_batch_attention_mask = pad_batch_attention_mask[:,-model.config.multimodal_max_length:]
+pad_batch_pixel_values = [item for sublist in batch_pixel_values for item in sublist]
+
+# generate output
+with torch.inference_mode():
+    gen_kwargs = dict(
+        max_new_tokens=1024,
+        do_sample=False,
+        top_p=None,
+        top_k=None,
+        temperature=None,
+        repetition_penalty=None,
+        eos_token_id=model.generation_config.eos_token_id,
+        pad_token_id=text_tokenizer.pad_token_id,
+        use_cache=True
+    )
+    output_ids = model.generate(pad_batch_input_ids, pixel_values=pad_batch_pixel_values, attention_mask=pad_batch_attention_mask, **gen_kwargs)
+
+for i in range(len(batch_input_ids)):
+    output = text_tokenizer.decode(output_ids[i], skip_special_tokens=True)
+    print(f'Output_{i}:\n{output}')
+```
+</details>
+
 ## Citation
 If you find Ovis useful, please cite the paper
 ```