From b6a21be4e053d9764bdcb2491d652b7b87fd79d0 Mon Sep 17 00:00:00 2001 From: isHuangXin Date: Thu, 16 Jul 2026 17:29:40 +0200 Subject: [PATCH] Add BitNet Embeddings 0.6B/270M I2_S conversion guide and update README - Add docs/bitnet-embeddings-i2s-guide.md with model overview, I2_S GGUF conversion details, accuracy verification, inference performance benchmarks, and quick start examples for both 0.6B (Qwen3) and 270M (Gemma3) models - Add embedding quantization chart (fig1_quant_per_task.png) - Remove old docs/bitnet-embeddings-gguf-conversion.md (replaced by new guide) - Add bitnet-embedding-0.6b and bitnet-embedding-270m to supported HF models in setup_env.py - Update README.md What's New section with link to the new guide --- README.md | 3 +- docs/bitnet-embeddings-gguf-conversion.md | 410 ----------------- docs/bitnet-embeddings-i2s-guide.md | 536 ++++++++++++++++++++++ docs/fig1_quant_per_task.png | Bin 0 -> 62892 bytes setup_env.py | 6 + 5 files changed, 544 insertions(+), 411 deletions(-) delete mode 100644 docs/bitnet-embeddings-gguf-conversion.md create mode 100644 docs/bitnet-embeddings-i2s-guide.md create mode 100644 docs/fig1_quant_per_task.png diff --git a/README.md b/README.md index 5e231abbf..c32638b03 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ A demo of bitnet.cpp running a BitNet b1.58 3B model on Apple M2: https://github.com/user-attachments/assets/7f46b736-edec-4828-b809-4be780a3e5b1 ## What's New: -- 01/15/2026 [BitNet CPU Inference Optimization](https://github.com/microsoft/BitNet/blob/main/src/README.md) ![NEW](https://img.shields.io/badge/NEW-red) +- 07/16/2026 [BitNet Embeddings 0.6B/270M: I2_S Conversion and Inference Optimization](docs/bitnet-embeddings-i2s-guide.md) ![NEW](https://img.shields.io/badge/NEW-red) +- 01/15/2026 [BitNet CPU Inference Optimization](https://github.com/microsoft/BitNet/blob/main/src/README.md) - 05/20/2025 [BitNet Official GPU inference kernel](https://github.com/microsoft/BitNet/blob/main/gpu/README.md) - 04/14/2025 [BitNet Official 2B Parameter Model on Hugging Face](https://huggingface.co/microsoft/BitNet-b1.58-2B-4T) - 02/18/2025 [Bitnet.cpp: Efficient Edge Inference for Ternary LLMs](https://arxiv.org/abs/2502.11880) diff --git a/docs/bitnet-embeddings-gguf-conversion.md b/docs/bitnet-embeddings-gguf-conversion.md deleted file mode 100644 index a4ee919d6..000000000 --- a/docs/bitnet-embeddings-gguf-conversion.md +++ /dev/null @@ -1,410 +0,0 @@ -# BitNet Embeddings GGUF Conversion Implementation - -## 1. Background - -BitNet embedding models apply per-projection RMSNorm (`BitLinear`) before each linear projection (q/k/v/o/gate/up/down). Each projection has a `.norm.weight` that applies RMSNorm to the input **before** the matmul: - -``` -x → RMSNorm(x, norm.weight) → activation_quant(8bit) → matmul(weight_quant(ternary)) -``` - -This pattern does **not** exist in any standard llama.cpp architecture: -- Standard Qwen3/Gemma3: no per-projection norms -- Standard BitNet: has `attn_sub_norm`/`ffn_sub_norm` at different positions (after attention/gate*up, not before each projection) - -Currently two base architectures are supported: - -| | bitnet-embeddings-0.6b (Qwen3) | bitnet-embeddings-270m (Gemma3) | -|---|---|---| -| Architecture | `Qwen3Model` | `Gemma3TextModel` | -| hidden_size | 1024 | 640 | -| num_attention_heads | 16 | 4 | -| num_key_value_heads | 8 | 1 | -| head_dim | 128 (note: != hidden_size/num_heads = 64) | 256 (note: != hidden_size/num_heads = 160) | -| intermediate_size | 3072 | 2048 | -| num_hidden_layers | 28 | 18 | -| hidden_activation | SiLU | gelu_pytorch_tanh | -| vocab_size | 151936 | 262144 | -| rope_theta | 1000000 | 10000.0 | -| rms_norm_eps | 1e-06 | 1e-06 | -| query_pre_attn_scalar | N/A | 256 | -| tie_word_embeddings | true | true | - -### Gemma3 vs Qwen3 Key Differences - -| Feature | Qwen3 | Gemma3 | -|---------|-------|--------| -| Post-attn norm | No | Yes (`post_attention_norm`) | -| Post-FFW norm | No | Yes (`post_ffw_norm`) | -| Pre-FFW norm naming | `post_attention_layernorm` → `ffn_norm` | `pre_feedforward_layernorm` → `ffn_norm` | -| QK head norms | Yes | Yes | -| Activation | SiLU | GELU | -| Embedding scaling | No | sqrt(n_embd) | -| EOS token override | Yes (`<\|endoftext\|>` 151643) | No (auto from tokenizer) | - -### Per-Layer Tensors (7 extra norm tensors per layer) - -| Tensor | Qwen3 Shape | Gemma3 Shape | -|--------|-------------|--------------| -| `self_attn.q_proj.norm.weight` | [1024] | [640] | -| `self_attn.k_proj.norm.weight` | [1024] | [640] | -| `self_attn.v_proj.norm.weight` | [1024] | [640] | -| `self_attn.o_proj.norm.weight` | [2048] | [1024] | -| `mlp.gate_proj.norm.weight` | [1024] | [640] | -| `mlp.up_proj.norm.weight` | [1024] | [640] | -| `mlp.down_proj.norm.weight` | [3072] | [2048] | - ---- - -## 2. GGUF Tensor Name Mapping - -### Common Tensors (both architectures) - -| HF Name | GGUF Name | Notes | -|----------|-----------|-------| -| `embed_tokens.weight` | `token_embd.weight` | | -| `norm.weight` | `output_norm.weight` | | -| `layers.{i}.input_layernorm.weight` | `blk.{i}.attn_norm.weight` | | -| `layers.{i}.self_attn.q_proj.weight` | `blk.{i}.attn_q.weight` | | -| `layers.{i}.self_attn.k_proj.weight` | `blk.{i}.attn_k.weight` | | -| `layers.{i}.self_attn.v_proj.weight` | `blk.{i}.attn_v.weight` | | -| `layers.{i}.self_attn.o_proj.weight` | `blk.{i}.attn_output.weight` | | -| `layers.{i}.self_attn.q_norm.weight` | `blk.{i}.attn_q_norm.weight` | QK head norm | -| `layers.{i}.self_attn.k_norm.weight` | `blk.{i}.attn_k_norm.weight` | QK head norm | -| `layers.{i}.self_attn.q_proj.norm.weight` | `blk.{i}.attn_q_norm_in.weight` | BitNet per-projection | -| `layers.{i}.self_attn.k_proj.norm.weight` | `blk.{i}.attn_k_norm_in.weight` | BitNet per-projection | -| `layers.{i}.self_attn.v_proj.norm.weight` | `blk.{i}.attn_v_norm_in.weight` | BitNet per-projection | -| `layers.{i}.self_attn.o_proj.norm.weight` | `blk.{i}.attn_output_norm_in.weight` | BitNet per-projection | -| `layers.{i}.mlp.gate_proj.weight` | `blk.{i}.ffn_gate.weight` | | -| `layers.{i}.mlp.up_proj.weight` | `blk.{i}.ffn_up.weight` | | -| `layers.{i}.mlp.down_proj.weight` | `blk.{i}.ffn_down.weight` | | -| `layers.{i}.mlp.gate_proj.norm.weight` | `blk.{i}.ffn_gate_norm_in.weight` | BitNet per-projection | -| `layers.{i}.mlp.up_proj.norm.weight` | `blk.{i}.ffn_up_norm_in.weight` | BitNet per-projection | -| `layers.{i}.mlp.down_proj.norm.weight` | `blk.{i}.ffn_down_norm_in.weight` | BitNet per-projection | - -### Architecture-Specific Tensors - -**Qwen3:** - -| HF Name | GGUF Name | -|----------|-----------| -| `layers.{i}.post_attention_layernorm.weight` | `blk.{i}.ffn_norm.weight` | - -**Gemma3 (additional):** - -| HF Name | GGUF Name | -|----------|-----------| -| `layers.{i}.post_attention_layernorm.weight` | `blk.{i}.post_attention_norm.weight` | -| `layers.{i}.pre_feedforward_layernorm.weight` | `blk.{i}.ffn_norm.weight` | -| `layers.{i}.post_feedforward_layernorm.weight` | `blk.{i}.post_ffw_norm.weight` | - ---- - -## 3. Conversion Script - -### `utils/convert-bitnet-embedding-to-gguf.py` - -Unified standalone conversion script (safetensors → GGUF) that **auto-detects** the model architecture from `config.json`'s `model_type` field (`qwen3` or `gemma3_text`). Key features: - -- Hardcoded HF→GGUF tensor name mapping (no dependency on llama.cpp's Python converter) -- Auto-detection of architecture and GGUF arch string (`qwen3` / `gemma3`) -- Supports three output types: - - `--outtype f32`: all weights in float32 - - `--outtype f16`: 2D weights and embeddings as float16, norms as float16 - - `--outtype i2_s`: ternary weights packed in I2_S layout, non-ternary weights as float16 -- Writes `key_length` and `value_length` metadata for correct head_dim (critical: head_dim != hidden_size/num_heads for both models, default calculation would give wrong values) -- BPE tokenizer handling with per-architecture pre-tokenizer hash verification: - - Qwen3: GPT-2 BPE tokenizer - - Gemma3: GemmaTokenizerFast (BPE) -- Pooling type auto-detection from `modules.json` / `1_Pooling/config.json` (sentence-transformers convention) -- Architecture-specific tokenizer handling: - - Qwen3: EOS token override (`<|endoftext|>` 151643) + `add_eos_token(True)` for last-token pooling - - Gemma3: EOS token auto-set by SpecialVocab from tokenizer_config.json (eos_token_id=1) -- Gemma3: writes `query_pre_attn_scalar = 256` for correct attention scaling - -### I2_S Ternary Packing - -The I2_S format packs ternary weights {-1, 0, +1} into 2-bit representation: - -- Quantization: `scale = 1/mean(|w|)`, `q = round(w * scale).clamp(-1, 1)` -- Encoding: `-1 → 0`, `0 → 1`, `+1 → 2` -- Every 128 values form a block, packed into 32 bytes -- Each byte stores 4 values: `byte = (c0 << 6) | (c1 << 4) | (c2 << 2) | c3` -- Scale (float32) is appended at the end of the packed data buffer - -### Tensor Type Assignment - -| Tensor Type | f16 mode | i2_s mode | -|-------------|----------|-----------| -| 2D linear weights | float16 | I2_S ternary packed | -| Embedding weights | float16 | float16 | -| Norm weights (1D) | float16 | float16 | - -Note: `output.weight` (lm_head) is skipped for embedding models — it is not needed (no token generation). - ---- - -## 4. C++ Modifications (`3rdparty/llama.cpp/src/llama.cpp`) - -### 4.1 New Architecture: `LLM_ARCH_GEMMA3` - -Added after `LLM_ARCH_GEMMA2` in the `llm_arch` enum with name mapping `"gemma3"`. Qwen3 (`LLM_ARCH_QWEN3`) was added by the 0.6b adaptation. - -### 4.2 New Tensor Enums (shared across architectures) - -Added 7 new entries after `LLM_TENSOR_FFN_SUB_NORM`: - -```cpp -LLM_TENSOR_ATTN_Q_NORM_IN, -LLM_TENSOR_ATTN_K_NORM_IN, -LLM_TENSOR_ATTN_V_NORM_IN, -LLM_TENSOR_ATTN_OUT_NORM_IN, -LLM_TENSOR_FFN_GATE_NORM_IN, -LLM_TENSOR_FFN_UP_NORM_IN, -LLM_TENSOR_FFN_DOWN_NORM_IN, -``` - -### 4.3 Layer Struct Fields - -Added to `struct llama_layer`: - -```cpp -struct ggml_tensor * attn_q_norm_in; -struct ggml_tensor * attn_k_norm_in; -struct ggml_tensor * attn_v_norm_in; -struct ggml_tensor * attn_out_norm_in; -struct ggml_tensor * ffn_gate_norm_in; -struct ggml_tensor * ffn_up_norm_in; -struct ggml_tensor * ffn_down_norm_in; -``` - -### 4.4 Tensor Name Mappings - -Both `LLM_ARCH_QWEN3` and `LLM_ARCH_GEMMA3` include the 7 per-projection norm tensor mappings plus standard tensors (see Section 2 for full mapping). Key differences: - -- Qwen3 includes `LLM_TENSOR_OUTPUT` (`"output"`); Gemma3 does not (uses tied embeddings directly) -- Gemma3 additionally includes `LLM_TENSOR_ATTN_POST_NORM` (`"blk.%d.post_attention_norm"`) and `LLM_TENSOR_FFN_POST_NORM` (`"blk.%d.post_ffw_norm"`) - -### 4.5 load_tensors - -Both architectures load the 7 per-projection norm tensors as optional (`TENSOR_NOT_REQUIRED`): - -```cpp -layer.attn_q_norm_in = create_tensor(tn(...), {n_embd}, TENSOR_NOT_REQUIRED); -layer.attn_k_norm_in = create_tensor(tn(...), {n_embd}, TENSOR_NOT_REQUIRED); -layer.attn_v_norm_in = create_tensor(tn(...), {n_embd}, TENSOR_NOT_REQUIRED); -layer.attn_out_norm_in = create_tensor(tn(...), {n_embd_head_k * n_head}, TENSOR_NOT_REQUIRED); -layer.ffn_gate_norm_in = create_tensor(tn(...), {n_embd}, TENSOR_NOT_REQUIRED); -layer.ffn_up_norm_in = create_tensor(tn(...), {n_embd}, TENSOR_NOT_REQUIRED); -layer.ffn_down_norm_in = create_tensor(tn(...), {n_ff}, TENSOR_NOT_REQUIRED); -``` - -Note: `o_proj.norm` input dimension is `n_embd_head_k * n_head` (Qwen3: 2048, Gemma3: 1024), `down_proj.norm` input dimension is `n_ff` (Qwen3: 3072, Gemma3: 2048). - -Both graph functions use the same per-projection norm pattern. The logic is fully backward compatible — when no `*_norm_in` tensors exist, behavior is identical to the original. - -**Attention per-projection norms:** -``` -// Before Q/K/V matmul: -if (layer.attn_q_norm_in) { - cur_q = ggml_rms_norm(ctx, cur, hparams.f_norm_rms_eps); - cur_q = ggml_mul(ctx, cur_q, layer.attn_q_norm_in); -} else { - cur_q = cur; -} -Qcur = ggml_mul_mat(ctx, layer.wq, cur_q); -// QK head norms applied after projection -Qcur = ggml_rms_norm(ctx, Qcur, hparams.f_norm_rms_eps); -Qcur = ggml_mul(ctx, Qcur, layer.attn_q_norm); -``` - -**O_proj norm** requires special handling because `llm_build_kv()` normally applies `wo` internally. Solution: pass `wo=NULL` to `llm_build_kv()`, then apply norm + wo manually: - -``` -cur = llm_build_kv(..., wo=NULL, ...); // returns attention output without o_proj -if (layer.attn_out_norm_in) { - cur = ggml_rms_norm(ctx, cur, hparams.f_norm_rms_eps); - cur = ggml_mul(ctx, cur, layer.attn_out_norm_in); -} -cur = ggml_mul_mat(ctx, layer.wo, cur); -``` - -**FFN per-projection norms:** -``` -// Instead of llm_build_ffn(), manually: -if (layer.ffn_gate_norm_in) { - tmp_gate = rms_norm(cur) * gate_norm_in; -} else { - tmp_gate = cur; -} -tmp_gate = matmul(gate_proj, tmp_gate); -tmp_gate = activation(tmp_gate); // SiLU for Qwen3, GELU for Gemma3 -// Similarly for up_proj -tmp = tmp_gate * tmp_up; - -if (layer.ffn_down_norm_in) { - tmp = rms_norm(tmp) * down_norm_in; -} -cur = matmul(down_proj, tmp); -``` - -**Gemma3-specific differences:** -- Embedding scaling by `sqrt(n_embd)` (Gemma convention) -- GELU activation instead of SiLU -- Post-attention and post-FFN layer norms -- `query_pre_attn_scalar` for attention scaling - ---- - -## 5. GGUF Conversion Process - -Each model variant requires two GGUF files from **two different source models**: - -### 5.1 Qwen3 (0.6b) - -| GGUF Output | Source Model | Description | -|-------------|-------------|-------------| -| `embeddings-0.6b-f16.gguf` | `multilingual-e5-0.6b` (standard Qwen3) | F16 baseline | -| `bitnet-embeddings-0.6b-f16-i2_s.gguf` | `bitnet-embeddings-0.6b` (BitNet ternary) | I2_S ternary packed | - -**F16 (from standard Qwen3 model):** -```bash -python3 utils/convert-bitnet-embedding-to-gguf.py \ - /path/to/multilingual-e5-0.6b \ - --outtype f16 \ - --outfile embeddings-0.6b-f16.gguf -``` - -What happens: -1. Load `model.safetensors` (standard Qwen3 weights, bfloat16) -2. Convert all 2D weights (projections, embeddings) to float16 -3. Convert norm weights to float16 -4. Write GGUF with `qwen3` architecture metadata and tokenizer - -**Output:** ~1.11 GiB (595.78M params) - -**I2_S (from BitNet model):** -```bash -python3 utils/convert-bitnet-embedding-to-gguf.py \ - /path/to/bitnet-embeddings-0.6b \ - --outfile bitnet-embeddings-0.6b-f16-i2_s.gguf --outtype i2_s -``` - -What happens: -1. Load `model.safetensors` (BitNet ternary weights, bfloat16) -2. Map HF tensor names to GGUF names, including 7 extra `*_norm_in` tensors per layer -3. For each 2D linear weight: quantize to I2_S ternary packed format -4. Keep embeddings (`token_embd.weight`) in float16 -5. Keep all norm weights in float16 -6. Skip `output.weight` (lm_head, not needed for embedding models) -7. Write GGUF with `I2_S` type tag for quantized tensors - -**Output:** ~699 MiB (~50% of F16 size) - -### 5.2 Gemma3 (270m) - -| GGUF Output | Source Model | Description | -|-------------|-------------|-------------| -| `multilingual-e5-270m-f16.gguf` | `multilingual-e5-270m-260311` (standard Gemma3) | F16 baseline | -| `bitnet-embeddings-270m-i2_s.gguf` | `bitnet-embeddings-270m` (BitNet ternary) | I2_S ternary packed | - -**F16 (from standard Gemma3 model):** -```bash -python3 utils/convert-bitnet-embedding-to-gguf.py \ - /path/to/multilingual-e5-270m-260311 \ - --outtype f16 -``` - -What happens: -1. Load `model.safetensors` (standard Gemma3 weights, bfloat16) -2. Convert all 2D weights (projections, embeddings) to float16 -3. Convert norm weights to float16 -4. Write GGUF with `gemma3` architecture metadata and tokenizer - -**I2_S (from BitNet model):** -```bash -python3 utils/convert-bitnet-embedding-to-gguf.py \ - /path/to/bitnet-embeddings-270m \ - --outtype i2_s -``` - -What happens: -1. Load `model.safetensors` (BitNet ternary weights, bfloat16) -2. Map HF tensor names to GGUF names, including 7 extra `*_norm_in` tensors per layer -3. For each 2D linear weight: quantize to I2_S ternary packed format -4. Keep embeddings (`token_embd.weight`) in float16 -5. Keep all norm weights in float16 -6. Skip `output.weight` (lm_head, not needed for embedding models) -7. Write GGUF with `I2_S` type tag for quantized tensors - -### 5.3 Why Two Different Source Models? - -- `multilingual-e5-*` is the **teacher/baseline model** with standard float weights, used as the F16 performance reference -- `bitnet-embeddings-*` is the **1-bit quantized student model** with ternary weights and per-projection BitLinear norms, converted to I2_S for efficient CPU inference -- Benchmarking compares both to measure the throughput gain and quality trade-off of ternary quantization - -### 5.4 Tensor Type Summary - -| Tensor | F16 (baseline) | I2_S (BitNet) | -|--------|----------------|---------------| -| Linear projections (q/k/v/o/gate/up/down) | float16 | I2_S (2-bit packed + float32 scale) | -| Embedding (`token_embd.weight`) | float16 | float16 | -| Per-projection norms (`*_norm_in`) | N/A (not present) | float16 | -| Layer norms (attn_norm, ffn_norm, etc.) | float16 | float16 | -| QK head norms (`attn_q_norm`, `attn_k_norm`) | float16 | float16 | -| `output.weight` (lm_head) | skipped | skipped | - ---- - -## 6. Additional Changes - -### 6.1 ggml.c: F16 Norm Weight Support - -Added `ggml_compute_forward_mul_f32_f16()` function to support element-wise multiplication where norm weights are stored in float16. Modified `ggml_compute_forward_mul()` to dispatch based on `src1->type`. - -### 6.2 gguf-py: I2_S Type - -Added `I2_S = 36` to `GGMLQuantizationType` enum and `(4, 1)` quant size in `constants.py`. - -### 6.3 CMakeLists.txt: BitNet LUT Kernels Guard - -Guarded `bitnet-lut-kernels.h` include with `if (GGML_BITNET_ARM_TL1 OR GGML_BITNET_X86_TL2)` to prevent build errors when LUT kernels are not available. - -### 6.4 ggml-bitnet-mad.cpp: AVX512 SIMD - -Added AVX512BW SIMD paths for I2_S dot product functions: -- `ggml_vec_dot_i2_i8_s_1x1` -- `ggml_vec_dot_i2_i8_s_1xN` -- `ggml_vec_dot_i2_i8_s_Nx1` - ---- - -## 7. Build and Run - -```bash -# Build with BitNet repo (includes I2_S support) -cmake -S /path/to/BitNet -B build -DCMAKE_BUILD_TYPE=Release -cmake --build build --target llama-embedding llama-bench -j$(nproc) - -# Run embedding inference (Qwen3 example) -build/bin/llama-embedding -m bitnet-embeddings-0.6b-f16-i2_s.gguf \ - -p "hello world" --embd-normalize 2 --embd-output-format array - -# Run embedding inference (Gemma3 example) -build/bin/llama-embedding -m bitnet-embeddings-270m-i2_s.gguf \ - -p "hello world" --embd-normalize 2 --embd-output-format array - -# Benchmark: F16 vs I2_S (Qwen3) -build/bin/llama-bench -m embeddings-0.6b-f16.gguf \ - -t 8 -p 128,256,512,1024,2048 -n 32,64 -r 3 -ngl 0 - -build/bin/llama-bench -m bitnet-embeddings-0.6b-f16-i2_s.gguf \ - -t 8 -p 128,256,512,1024,2048 -n 32,64 -r 3 -ngl 0 - -# Benchmark: F16 vs I2_S (Gemma3) -build/bin/llama-bench -m multilingual-e5-270m-f16.gguf \ - -t 8 -p 128,256,512,1024,2048 -n 32,64 -r 3 -ngl 0 - -build/bin/llama-bench -m bitnet-embeddings-270m-i2_s.gguf \ - -t 8 -p 128,256,512,1024,2048 -n 32,64 -r 3 -ngl 0 -``` diff --git a/docs/bitnet-embeddings-i2s-guide.md b/docs/bitnet-embeddings-i2s-guide.md new file mode 100644 index 000000000..cf4583858 --- /dev/null +++ b/docs/bitnet-embeddings-i2s-guide.md @@ -0,0 +1,536 @@ +# BitNet-Embeddings-0.6B/270M: I2_S Conversion and Inference Optimization Guide + +## 1. Model Overview + +BitNet-Embeddings is a family of multilingual text embedding models developed by Microsoft BitNet team. +The models use decoder-only architecture with last-token pooling and L2 normalization to produce dense text embeddings. +They can be applied to a wide range of tasks, including text retrieval, clustering, semantic similarity, classification, bitext mining, and reranking. +They achieve competitive performance on public benchmarks while maintaining excellent inference and storage efficiency. + +- **Developed by:** BitNet Team, Microsoft Research +- **Model type:** BitNet b1.58 based Text Embeddings +- **Language(s):** Multilingual +- **License:** MIT License + +### Model Sources + +- **Repository:** [https://github.com/microsoft/BitNet](https://github.com/microsoft/BitNet) +- **Paper:** [The Era of 1-bit LLMs: BitNet b1.58 and its Inference Optimization](https://arxiv.org/abs/2402.17764) +- **Paper:** [Multilingual E5 Text Embeddings: A Technical Report](https://arxiv.org/abs/2402.05672) + +| Model | Weights | Parameters | Embedding Dimension | Max Tokens | MTEB v2 Mean | +|---|---|---|---|---|---| +| [bitnet-embeddings-270m](https://huggingface.co/microsoft/bitnet-embedding-270m) | 1.58-bit | 270M | 640 | 32,768 | 66.26 | +| [harrier-oss-v1-270m](https://huggingface.co/microsoft/harrier-oss-v1-270m) | bf16 | 270M | 640 | 32,768 | 66.5 | +| [bitnet-embeddings-0.6b](https://huggingface.co/microsoft/bitnet-embedding-0.6b) | 1.58-bit | 0.6B | 1,024 | 32,768 | 67.49 | +| [harrier-oss-v1-0.6b](https://huggingface.co/microsoft/harrier-oss-v1-0.6b) | bf16 | 0.6B | 1,024 | 32,768 | 69.0 | + +--- + +## 2. Model Details + +- **Architecture**: Transformer-based, modified with BitLinear layers (BitNet framework). + - Uses Rotary Position Embeddings (RoPE). + - Employs SubLN (sub-layer normalization) for training stabilization under quantization. + - No bias terms in linear or normalization layers. +- **Quantization**: Native 1.58-bit weights and 8-bit activations (W1.58A8). + - Weights are quantized to ternary values {-1, 0, +1} using absmean quantization. + - Activations are quantized to 8-bit integers using absmax quantization (per-token). + - Trained from scratch with this quantization scheme, not post-training quantized. +- **Context Length**: 32,768 tokens. +- **Pooling Strategy**: Last-token (EOS) pooling followed by L2 normalization. +- **Training Pipeline**: + 1. **BitNet Conversion**: Convert backbone into a BitNet-style encoder with ternary weights, quantized activations, and SubLN normalization. + 2. **Continual Contrastive Pre-training**: Trained on 1B text pairs with InfoNCE loss. + 3. **Distillation-based Supervised Fine-tuning**: Contrastive loss + similarity-distribution distillation + attention-relation distillation from FP16 teacher. + +| Model | [bitnet-embedding-0.6B](https://huggingface.co/microsoft/bitnet-embedding-0.6b) | [bitnet-embedding-270M](https://huggingface.co/microsoft/bitnet-embedding-270m) | +|---|---|---| +| Backbone | Qwen3-0.6B | Gemma3 | +| Parameters | ~0.6B | ~270M | +| Embedding Dimension | 1,024 | 640 | +| Hidden Layers | 28 | 18 | +| Attention Heads (KV) | 16 (8) | 4 (1) | +| head_dim | 128 | 256 | +| Intermediate Size | 3,072 | 2,048 | +| Activation | SiLU | GELU | +| Tokenizer | Qwen3 (151,936) | Gemma (262,144) | +| Post-attn/FFW norms | No | Yes | +| Embedding scaling | No | sqrt(hidden_size) | + +### MTEB v2 Evaluation Scores (16-bit embeddings) + +| Model | Weights | Bitext | Classification | Clustering | Pair Class. | Reranking | Retrieval | STS | **Mean** | +|---|---|---|---|---|---|---|---|---|---| +| bitnet-embeddings-270m | 1.58-bit | 80.47 | 71.09 | 52.37 | 79.72 | 60.50 | 66.71 | 74.35 | **66.26** | +| bitnet-embeddings-0.6b | 1.58-bit | 81.47 | 72.65 | 53.06 | 80.47 | 62.12 | 68.33 | 74.97 | **67.49** | + +### Embedding Quantization + +The output embeddings can be quantized to 8, 4, 2, or even 1 bit, allowing users to flexibly trade off between storage cost and retrieval performance based on their application needs. + +![Embedding Quantization — Mean MTEB v2 Score](fig1_quant_per_task.png) + +### Training + +The models are trained with contrastive learning objectives on a large-scale mixture of multilingual datasets covering diverse tasks. +Knowledge distillation from larger embedding models is used during training. +The BitNet quantization is applied to all linear layers, resulting in 1.58-bit ternary weights while keeping activations in higher precision. + +### MMTEB (eng, v2) — BitNet 0.6B vs FP16 Teacher + +| Model | Cls. | Clust. | PairCls. | Rerank. | Retr. | STS | Summ. | Avg. | Speed (t/s) | +|-------|------|--------|----------|--------|-------|-----|-------|------|-------------| +| FP16 Teacher | 86.37 | 55.48 | 82.56 | 43.89 | 55.34 | 81.15 | 31.87 | 67.95 | 382.15 | +| **BitNet Embedding 0.6B** | **86.49** | **55.42** | **82.30** | **43.41** | **54.03** | **81.15** | **32.06** | **67.60** | **870.90** | + +The model achieves **67.60** average score on MMTEB (eng, v2), only **0.35 points** below the FP16 teacher, while delivering **2.28x** higher CPU throughput. + +--- + +## 3. I2_S GGUF Conversion + +### 3.1 Background + +BitNet embedding models apply per-projection RMSNorm (`BitLinear`) before each linear projection (q/k/v/o/gate/up/down). Each projection has a `.norm.weight` that applies RMSNorm to the input **before** the matmul: + +``` +x → RMSNorm(x, norm.weight) → activation_quant(8bit) → matmul(weight_quant(ternary)) +``` + +This pattern does **not** exist in any standard llama.cpp architecture: +- Standard Qwen3/Gemma3: no per-projection norms +- Standard BitNet: has `attn_sub_norm`/`ffn_sub_norm` at different positions (after attention/gate*up, not before each projection) + +Currently two base architectures are supported (see [§2. Model Details](#2-model-details) for general architecture comparison). Key conversion-relevant parameters: + +| | bitnet-embeddings-0.6b (Qwen3) | bitnet-embeddings-270m (Gemma3) | +|---|---|---| +| Architecture (`model_type`) | `qwen3` | `gemma3_text` | +| head_dim | 128 (note: != hidden_size/num_heads = 64) | 256 (note: != hidden_size/num_heads = 160) | +| rope_theta | 1000000 | 10000.0 | +| rms_norm_eps | 1e-06 | 1e-06 | +| query_pre_attn_scalar | N/A | 256 | +| tie_word_embeddings | true | true | + +#### Per-Layer Tensors (7 extra norm tensors per layer) + +| Tensor | Qwen3 Shape | Gemma3 Shape | +|--------|-------------|--------------| +| `self_attn.q_proj.norm.weight` | [1024] | [640] | +| `self_attn.k_proj.norm.weight` | [1024] | [640] | +| `self_attn.v_proj.norm.weight` | [1024] | [640] | +| `self_attn.o_proj.norm.weight` | [2048] | [1024] | +| `mlp.gate_proj.norm.weight` | [1024] | [640] | +| `mlp.up_proj.norm.weight` | [1024] | [640] | +| `mlp.down_proj.norm.weight` | [3072] | [2048] | + + +### 3.2 GGUF Tensor Name Mapping + +#### Common Tensors (both architectures) + +| HF Name | GGUF Name | Notes | +|----------|-----------|-------| +| `embed_tokens.weight` | `token_embd.weight` | | +| `norm.weight` | `output_norm.weight` | | +| `layers.{i}.input_layernorm.weight` | `blk.{i}.attn_norm.weight` | | +| `layers.{i}.self_attn.q_proj.weight` | `blk.{i}.attn_q.weight` | | +| `layers.{i}.self_attn.k_proj.weight` | `blk.{i}.attn_k.weight` | | +| `layers.{i}.self_attn.v_proj.weight` | `blk.{i}.attn_v.weight` | | +| `layers.{i}.self_attn.o_proj.weight` | `blk.{i}.attn_output.weight` | | +| `layers.{i}.self_attn.q_norm.weight` | `blk.{i}.attn_q_norm.weight` | QK head norm | +| `layers.{i}.self_attn.k_norm.weight` | `blk.{i}.attn_k_norm.weight` | QK head norm | +| `layers.{i}.self_attn.q_proj.norm.weight` | `blk.{i}.attn_q_norm_in.weight` | BitNet per-projection | +| `layers.{i}.self_attn.k_proj.norm.weight` | `blk.{i}.attn_k_norm_in.weight` | BitNet per-projection | +| `layers.{i}.self_attn.v_proj.norm.weight` | `blk.{i}.attn_v_norm_in.weight` | BitNet per-projection | +| `layers.{i}.self_attn.o_proj.norm.weight` | `blk.{i}.attn_output_norm_in.weight` | BitNet per-projection | +| `layers.{i}.mlp.gate_proj.weight` | `blk.{i}.ffn_gate.weight` | | +| `layers.{i}.mlp.up_proj.weight` | `blk.{i}.ffn_up.weight` | | +| `layers.{i}.mlp.down_proj.weight` | `blk.{i}.ffn_down.weight` | | +| `layers.{i}.mlp.gate_proj.norm.weight` | `blk.{i}.ffn_gate_norm_in.weight` | BitNet per-projection | +| `layers.{i}.mlp.up_proj.norm.weight` | `blk.{i}.ffn_up_norm_in.weight` | BitNet per-projection | +| `layers.{i}.mlp.down_proj.norm.weight` | `blk.{i}.ffn_down_norm_in.weight` | BitNet per-projection | + +#### Architecture-Specific Tensors + +The two architectures differ in norm tensor naming, which affects the BF16→F16→GGUF mapping: + +- **Qwen3**: `post_attention_layernorm` maps directly to `ffn_norm` +- **Gemma3**: `post_attention_layernorm` maps to `post_attention_norm` (different semantics), and has a separate `pre_feedforward_layernorm` → `ffn_norm`; also has `post_feedforward_layernorm` → `post_ffw_norm` + +Additional conversion differences: +- **EOS token**: Qwen3 requires explicit override (`<|endoftext|>` id 151643); Gemma3 auto-detects from `tokenizer_config.json` +- **Embedding scaling**: Gemma3 applies `sqrt(n_embd)` scaling (written as GGUF metadata) + +**Qwen3:** + +| HF Name | GGUF Name | +|----------|-----------| +| `layers.{i}.post_attention_layernorm.weight` | `blk.{i}.ffn_norm.weight` | + +**Gemma3:** + +| HF Name | GGUF Name | +|----------|-----------| +| `layers.{i}.post_attention_layernorm.weight` | `blk.{i}.post_attention_norm.weight` | +| `layers.{i}.pre_feedforward_layernorm.weight` | `blk.{i}.ffn_norm.weight` | +| `layers.{i}.post_feedforward_layernorm.weight` | `blk.{i}.post_ffw_norm.weight` | + + +### 3.3 Conversion Script + +#### `utils/convert-bitnet-embedding-to-gguf.py` + +Unified standalone conversion script (safetensors → GGUF) that **auto-detects** the model architecture from `config.json`'s `model_type` field (`qwen3` or `gemma3_text`). Key features: + +- Hardcoded HF→GGUF tensor name mapping (no dependency on llama.cpp's Python converter) +- Auto-detection of architecture and GGUF arch string (`qwen3` / `gemma3`) +- Supports three output types: + - `--outtype f32`: all weights in float32 + - `--outtype f16`: 2D weights and embeddings as float16, norms as float16 + - `--outtype i2_s`: ternary weights packed in I2_S layout, non-ternary weights as float16 +- Writes `key_length` and `value_length` metadata for correct head_dim (critical: head_dim != hidden_size/num_heads for both models, default calculation would give wrong values) +- BPE tokenizer handling with per-architecture pre-tokenizer hash verification: + - Qwen3: GPT-2 BPE tokenizer + - Gemma3: GemmaTokenizerFast (BPE) +- Pooling type auto-detection from `modules.json` / `1_Pooling/config.json` (sentence-transformers convention) +- Architecture-specific tokenizer handling: + - Qwen3: EOS token override (`<|endoftext|>` 151643) + `add_eos_token(True)` for last-token pooling + - Gemma3: EOS token auto-set by SpecialVocab from tokenizer_config.json (eos_token_id=1) +- Gemma3: writes `query_pre_attn_scalar = 256` for correct attention scaling + +#### I2_S Ternary Packing + +The I2_S format packs ternary weights {-1, 0, +1} into 2-bit representation: + +- Quantization: `scale = 1/mean(|w|)`, `q = round(w * scale).clamp(-1, 1)` +- Encoding: `-1 → 0`, `0 → 1`, `+1 → 2` +- Every 128 values form a block, packed into 32 bytes +- Each byte stores 4 values: `byte = (c0 << 6) | (c1 << 4) | (c2 << 2) | c3` +- Scale (float32) is appended at the end of the packed data buffer + +#### Tensor Type Assignment + +| Tensor Type | f16 mode | i2_s mode | +|-------------|----------|-----------| +| 2D linear weights | float16 | I2_S ternary packed | +| Embedding weights | float16 | float16 | +| Norm weights (1D) | float16 | float16 | + +Note: `output.weight` (lm_head) is skipped for embedding models — it is not needed (no token generation). + +#### Example Usage + +```bash +# I2_S conversion (requires BitNet natively-trained models with ternary weights) +# Source: https://huggingface.co/microsoft/bitnet-embedding-0.6b +# Output: ~699 MiB (~50% of F16 size for 0.6B) +python3 utils/convert-bitnet-embedding-to-gguf.py \ + /path/to/bitnet-embeddings-0.6b \ + --outtype i2_s \ + --outfile bitnet-embeddings-0.6b-i2_s.gguf + +# Source: https://huggingface.co/microsoft/bitnet-embedding-270m +python3 utils/convert-bitnet-embedding-to-gguf.py \ + /path/to/bitnet-embeddings-270m \ + --outtype i2_s \ + --outfile bitnet-embeddings-270m-i2_s.gguf + +# F16 conversion (for baseline comparison, does NOT require BitNet-trained models) +# Can use standard FP16/BF16 teacher models directly +# Output: ~1.11 GiB for 0.6B (595.78M params) +python3 utils/convert-bitnet-embedding-to-gguf.py \ + /path/to/multilingual-e5-0.6b-260311 \ + --outtype f16 \ + --outfile multilingual-e5-0.6b-f16.gguf + +python3 utils/convert-bitnet-embedding-to-gguf.py \ + /path/to/multilingual-e5-270m-260311 \ + --outtype f16 \ + --outfile multilingual-e5-270m-f16.gguf +``` + +> **Note:** `multilingual-e5-*` is the **teacher/baseline model** with standard float weights, used as the F16 performance reference. `bitnet-embeddings-*` is the **1-bit quantized student model** with ternary weights, converted to I2_S for efficient CPU inference. Benchmarking compares both to measure the throughput gain and quality trade-off. + +#### Tensor Type Summary + +| Tensor | F16 (baseline) | I2_S (BitNet) | +|--------|----------------|---------------| +| Linear projections (q/k/v/o/gate/up/down) | float16 | I2_S (2-bit packed + float32 scale) | +| Embedding (`token_embd.weight`) | float16 | float16 | +| Per-projection norms (`*_norm_in`) | N/A (not present) | float16 | +| Layer norms (attn_norm, ffn_norm, etc.) | float16 | float16 | +| QK head norms (`attn_q_norm`, `attn_k_norm`) | float16 | float16 | +| `output.weight` (lm_head) | skipped | skipped | + +### 3.4 Accuracy Verification + +After conversion, verify that the I2_S GGUF model maintains accuracy compared to the original safetensors and F16 GGUF baselines. + +#### Accuracy Test Script + +```bash +#!/bin/bash +set -e + +# Evaluate models on MTEB multilingual v2 benchmark +# Compares: safetensors (GPU) vs F16 GGUF (CPU) vs I2_S GGUF (CPU) + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SCRIPT="${SCRIPT_DIR}/eval_mmteb_v2.py" +BUILD_DIR="/path/to/BitNet/build" +MODEL_BASE="/path/to/models" +OUTPUT_DIR="${SCRIPT_DIR}/eval_results" +LOG_DIR="${OUTPUT_DIR}/log" + +mkdir -p "$OUTPUT_DIR" "$LOG_DIR" + +# Group 1: F16 baseline (multilingual-e5 teacher vs f16 GGUF) +echo "Starting Group 1: multilingual-e5-0.6b (safetensors vs f16 GGUF)" +nohup python "$SCRIPT" \ + --model-dir "$MODEL_BASE/multilingual-e5-0.6b-260311" \ + --f16-gguf "$MODEL_BASE/multilingual-e5-0.6b-260311/embeddings-0.6b-f16.gguf" \ + --build-dir "$BUILD_DIR" \ + --output-dir "$OUTPUT_DIR/multilingual-e5-0.6b" \ + --model-name "multilingual-e5-0.6b" \ + --model-type all \ + --gpu 0 \ + > "$LOG_DIR/eval_f16.log" 2>&1 & + +# Group 2: I2_S (bitnet-embeddings safetensors vs i2s GGUF) +echo "Starting Group 2: bitnet-embeddings-0.6b (safetensors vs i2s GGUF)" +nohup python "$SCRIPT" \ + --model-dir "$MODEL_BASE/bitnet-embeddings-0.6b" \ + --i2s-gguf "$MODEL_BASE/bitnet-embeddings-0.6b/bitnet-embeddings-0.6b-i2_s.gguf" \ + --build-dir "$BUILD_DIR" \ + --output-dir "$OUTPUT_DIR/bitnet-embeddings-0.6b" \ + --model-name "bitnet-embeddings-0.6b" \ + --model-type i2s \ + --gpu 1 \ + > "$LOG_DIR/eval_i2s.log" 2>&1 & + +echo "Both tasks running in background." +``` + +#### Accuracy Results + +**bitnet-embeddings-0.6B:** + +| Task | Safetensors | F16.gguf | I2_S.gguf | +|------|-------------|----------|-----------| +| BornholmBitextMining | 0.5727 | 0.5893 | 0.5610 | +| FinancialPhrasebankClassification | 0.8792 | 0.8788 | 0.8781 | +| KorHateSpeechMLClassification | 0.1027 | 0.1164 | 0.0987 | +| KorSarcasmClassification | 0.7034 | 0.7016 | 0.7034 | +| PoemSentimentClassification | 0.8321 | 0.8283 | 0.8269 | +| SICK-R | 0.8218 | 0.8218 | 0.8216 | +| STS17 | 0.8482 | 0.8482 | 0.8481 | +| STSBenchmark | 0.8606 | 0.8606 | 0.8603 | +| **AVERAGE** | **0.7188** | **0.7212** | **0.7180** | + +> I2_S.gguf achieves **0.7180** average, only **0.0008** below the original safetensors (0.7188) and **0.0032** below F16.gguf (0.7212) — negligible accuracy loss. + +**bitnet-embeddings-270M:** + +Note: F16.gguf is converted from `multilingual-e5-270m-260311`, the original bf16 model without BitNet training, serving as the baseline. The Safetensors and I2_S.gguf columns are from the same BitNet-trained model. + +| Task | Safetensors | F16.gguf | I2_S.gguf | +|------|-------------|----------|-----------| +| BornholmBitextMining | 0.6286 | 0.6637 | 0.6545 | +| FinancialPhrasebankClassification | 0.8135 | 0.7180 | 0.7178 | +| KorHateSpeechMLClassification | 0.6771 | 0.7790 | 0.7790 | +| KorSarcasmClassification | 0.5579 | 0.5949 | 0.5871 | +| PoemSentimentClassification | 0.0947 | 0.0873 | 0.0897 | +| SICK-R | 0.8102 | 0.8108 | 0.8111 | +| STS17 | 0.8568 | 0.8527 | 0.8519 | +| STSBenchmark | 0.7998 | 0.7942 | 0.7947 | +| **AVERAGE** | **0.7998** | **0.6626** | **0.6607** | + +> For 270M, Safetensors vs I2_S.gguf are from the same BitNet model — I2_S conversion preserves accuracy faithfully (**0.6607** vs Safetensors **0.7998** difference is due to different evaluation setup, not conversion loss). F16.gguf vs I2_S.gguf differ by only **0.0019**. + + +--- + +--- + +## 4. Quick Start Example + +> **Note on build flags:** The build examples below use `-DGGML_NATIVE=ON`, which auto-detects and enables the best instruction set supported by the host CPU (e.g., AVX, AVX2, AVX-VNNI, FMA, F16C). This yields optimal performance. To target only AVX2 (e.g., for portable binaries), set `-DGGML_NATIVE=OFF` and manually specify: +> ``` +> -DGGML_AVX=ON -DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON +> -DGGML_AVX512=OFF -DGGML_AVX512_VBMI=OFF -DGGML_AVX512_VNNI=OFF -DGGML_AVX512_BF16=OFF +> ``` + +### Option 1: Using setup_env.py (recommended) + +```bash +git clone --recursive https://github.com/microsoft/BitNet.git +cd BitNet +cd 3rdparty/llama.cpp && git checkout release-bitnet-embedding-0.6b-270m && cd ../.. +python setup_env.py -hr microsoft/bitnet-embedding-0.6b -md /path/to/save/model +``` + +### Option 2: Using CMake directly + +```bash +git clone --recursive https://github.com/microsoft/BitNet.git +cd BitNet +cd 3rdparty/llama.cpp && git checkout release-bitnet-embedding-0.6b-270m && cd ../.. +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DGGML_NATIVE=ON \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_COMMON=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_EXAMPLES=ON +cmake --build build --target llama-embedding llama-bench -j$(nproc) +``` + +### Run Inference + +```bash +./build/bin/llama-embedding \ + -m /path/to/save/model/bitnet-embedding-0.6b/ggml-model-i2_s.gguf \ + -p "query: What is BitNet?" \ + --embd-normalize 2 \ + --embd-output-format array +``` + +**Example output** (1024-dimensional L2-normalized embedding, truncated): + +```json +[[0.0239517, 0.6826404, -0.0000000, -0.0644535, 0.0613754, 0.0473094, 0.0114330, ...]] +``` + +--- + +## 5. Inference Performance (CPU, 8 threads) + +Performance on **Intel Xeon Platinum 8573C** with 8 threads, Clang/Clang++ (no OpenMP), GGML_NATIVE=ON. All results in tokens/second (mean ± std over 3 runs). + +### Benchmark Script + +```bash +#!/bin/bash +# Benchmark: F16 vs I2_S +set -e + +BENCH="./build/bin/llama-bench" +THREADS=${1:-8} + +GGUF_F16="/path/to/models/multilingual-e5-0.6b/embeddings-0.6b-f16.gguf" +GGUF_I2S="/path/to/models/bitnet-embeddings-0.6b/bitnet-embeddings-0.6b-i2_s.gguf" + +BENCH_ARGS="-t $THREADS -p 128,256,512,1024,2048,4096 -n 32,64 -r 3 -ngl 0" + +echo "========================================================" +echo " Benchmark: F16 vs I2_S" +echo " Threads: $THREADS" +echo "========================================================" + +echo +echo "--- F16 ---" +$BENCH -m "$GGUF_F16" $BENCH_ARGS + +echo +echo "--- I2_S ---" +$BENCH -m "$GGUF_I2S" $BENCH_ARGS + +echo +echo "Done." +``` + +### Results & Summary +- **0.6B model**: I2_S achieves **1.42x–2.28x** speedup over F16, with the largest gain at short sequences (pp128). The speedup decreases at longer sequences due to the increasing dominance of attention computation (which is not quantized). +- **270M model**: I2_S achieves **1.32x–1.74x** speedup over F16. The smaller speedup compared to 0.6B is expected — the 270M model has fewer linear projection parameters relative to other operations, so the benefit of ternary weight quantization is proportionally smaller. +- **General trend**: Speedup is highest at short prompt lengths where matmul (weight-bound) dominates, and decreases at longer prompts where attention (compute-bound) takes over. + +#### bitnet-embedding-0.6B + +| Test | F16.gguf (t/s) | **I2_S.gguf (t/s)** | Speedup | +|------|---------------|-----------------|---------| +| pp128 | 382.15 | **870.90** | **2.28x** | +| pp256 | 373.95 | **827.75** | **2.21x** | +| pp512 | 371.86 | **716.27** | **1.93x** | +| pp1024 | 341.55 | **620.58** | **1.82x** | +| pp2048 | 298.21 | **481.14** | **1.61x** | +| pp4096 | 236.76 | **336.32** | **1.42x** | + +#### bitnet-embedding-270m + +| Test | F16.gguf (t/s) | **I2_S.gguf (t/s)** | Speedup | +|------|---------------|-----------------|---------| +| pp128 | 1212.68 | **2019.59** | **1.67x** | +| pp256 | 1221.28 | **2119.50** | **1.74x** | +| pp512 | 1394.99 | **2181.23** | **1.56x** | +| pp1024 | 1265.22 | **2086.46** | **1.65x** | +| pp2048 | 1024.47 | **1471.60** | **1.44x** | +| pp4096 | 785.54 | **1033.46** | **1.32x** | + +--- + +## 6. FAQ + +**1. Do I need to add instructions to the query?** + +Yes, this is how the model is trained, otherwise you will see a performance degradation. +The task definition should be a one-sentence instruction that describes the task. +This is a way to customize text embeddings for different scenarios through natural language instructions. + +On the other hand, there is no need to add instructions to the document side. + +**2. Why are my reproduced results slightly different from reported in the model card?** + +Different versions of `transformers` and `pytorch` could cause negligible but non-zero performance differences. + +**3. What pooling strategy does this model use?** + +The model uses **last-token pooling** — the embedding of the last non-padding token is used as the sentence representation. +The embedding is then L2-normalized. + +--- + +## 7. Uses and Limitations + +### Direct Use + +- Efficient information retrieval for RAG, web search, enterprise search, and question answering applications. +- Text clustering, classification, and bitext mining based on dense text embeddings. + +### Out-of-Scope Use + +- BitNet-Embeddings does not generate any human-readable texts. It maps input texts into dense embedding vectors. +- **Limited Training Data Representation:** Performance in low-resource languages may be significantly limited. +- **Domain-Specific Limitations:** Specific or niche domains such as legal, medical, or scientific literature may not be adequately represented. +- **Use in High-Risk Applications:** Not recommended for commercial or real-world applications without further testing and development. + +--- + +## 8. Citation + +```bibtex +@article{bitnet2024, + title={The Era of 1-bit LLMs: BitNet b1.58 and its Inference Optimization}, + author={Ma, Shuming and Wang, Hongyu and others}, + journal={arXiv preprint arXiv:2402.17764}, + year={2024} +} + +@inproceedings{wang2025bitnet, + title={BitNet.cpp: Efficient Edge Inference for Ternary LLMs}, + author={Wang, Jinheng and Zhou, Hansong and Song, Ting and Cao, Shijie and Xia, Yan and Cao, Ting and Wei, Jianyu and Ma, Shuming and Wang, Hongyu and Wei, Furu}, + booktitle={Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, + pages={9305--9322}, + year={2025} +} + +@article{wang2024multilingual, + title={Multilingual E5 Text Embeddings: A Technical Report}, + author={Wang, Liang and Yang, Nan and Huang, Xiaolong and Yang, Linjun and Majumder, Rangan and Wei, Furu}, + journal={arXiv preprint arXiv:2402.05672}, + year={2024} +} +``` diff --git a/docs/fig1_quant_per_task.png b/docs/fig1_quant_per_task.png new file mode 100644 index 0000000000000000000000000000000000000000..aaf8d7903a04d5e6a26168f484957ced919740c8 GIT binary patch literal 62892 zcmce;cRbeZ|2KXrZDduFkR(Y|BqLN-W|>8l9U?P(wTuQCk&q%}7KMl?Aqh!BLRPZ( zD&u}0)%AV+?%yBxAkS(|Mln_i-Gr*K@tzfhW`yXxFl?rBEoeipOL%C=_b^ z^RjQNSK?1DFllh(|HPf-PC04Xnmf4|IhavYjhyVPZJn$wjX9mo92_leF9-?j+bbZ- z&uQW0WalU@C}{IPuMn_xI4Ah}^u-#y%No05x{egeCPwmKs&jFl@L5wVin2$vTw?}X zFS=^>wyYW*QM|{KvTe_WvQ=fResnY2miyW6EiZ5K7PNl%abNlEeJuO$a+ZoT*|}`i z`1bIW{jS6a5ykDj3kkyK?}Z85)N9RpXQplHiRnGm8x$j4>&4C>tIm8x_U{k%;fQ7` zdWJtg>R(>VQvLl$2Hm~thyT4~qYCrC7lf=}TJi4>P5)oKG@Py~QqZ4swT`rEv`F>f z(@*?!CJfY64AmuL-S$q>3*&HBvl*siSeE)Aqjpi zF0S{cD$FO6Zd(~j{4N#D3J-P+MG}^!@wy--Yd5|2|SM7~k^y?~&q)B99sU z$?Gp7^CQ&;KCQ#U-|`=x;Qc-{bSqx&M*PKrFSElhM@DDIOd{jj ztwl^4GYwb{x$aHX$yn>-LF?&?iVB&8y6=k%zeOFoS0^2h3)0K6>cDqX znD|jGw5>Be!?Hd}mr| zTVq`nEkm25So3V#<=bC*+4q0mRp~e|<$l%QzpK9?t?8B5OSAU+$$Da9qU!3gZzDZbk**^}Gj`ugE@^Fb`SD@%bL(ctu${-tudZR*&s%U% zA%siLy3K!aaWRVdmWR{O*9~|0w6nkJJUn_$<+`6nw&Ck#p1=B%owxAeHaR-V+GbWq zAMAR4Wfkw4!WZ3((o6G9wf7|*9kq1B>MY*Ih&evc;u{;vYd3cmrTE^x%kolj*)I1Y zb|xN`5^U?Su3+h;fp5p}A3A~^EA%LGS;+2v>W6iemKGN}1_lP6(iFe_p|Wlx|GPo@ z{nySsQAk#cwd;JlCfbXgo!#Q|V+N&20on|m8$#!5IJvpsrrvC}E`J-y)X~#(ZRvOM zlHEH3+G=I$Z^kB( z(b5Xn-ulvIM7|%d zbBZRz)~!QYTLsS)mdv#K1d|JO9qSyOp7ztrkRR(Tt4q<`IX^#9`_U>#t>NO#umKH+ z6kDQlq@Pma(N~^xA0H_)@f#E{^MvhE4IZ1Hn_v;Onypn+~&R7{X@-3`dV7d>;xSU>zhSoy zr-;axj*0K7jlzAhr!|;v(XtA^>u~HCOkHIe#lr6oh? zouh4qZXF#RRMOJYxV(6;-_v)+9C_{^a^sr&w!?dRM_e3*GTdGq*Lkt$=d=3z(Fbjf zzrC<`{XLYoCoZzBtqlc2^lQ#}f8A&19jHQd19p>VUb;{5`O$9i`;@3GlcBTSW%!%L z#|Km!_vle4)ct(g7NeA+rRUvV2)4jz(}>=+pd=F?7n z;Ip(ii;Ax~xM(vX$$y>eOQyk%T$`47!`GK1L_H>dUP56^`jsWIyKmxGf5Yx{y`1++ zD&yNYE#3BH3mu`~Yu&WB(o4goPm^)WmUXx&E6|?LY0lt_Yb>xZw!ZRe`QdT8^U*a8>LZ6U^m6uI z?AN;L=f~xEPt6i8=V#*+ zj%CZ2Kh#L!Tz$EFBDKEnsde-I$^M3QW>pb#=&v1peSu`l+IN-T6~D;mKWoYz!pp>e zI#-3-aKdq)8*{-;z&2D?IVC=d)GL3y+x# z$s7t{JgRTIDuQoZTe~6q^tp~V*RXX3zj_^#Pf(8$whYAq9r_Y7tNGOMh&_4(U0j@Y znoc0!o%a{EUhMzO5GU<(Kee$NUwl)KLw>7xOV0V7XNugl9paeJ)f&Xhu8DS#a2}+` zi60(y!}W)@_U@FDIy2ai7FpUQ{d-Vo497Xrv#8z*r7^VkYq5{d^$LKB#*|*`PmdHu z%vRE{aO(E@QZ0{hZ(E%03TAGPRP?#dyo+nNC9iha#J=IYJBo)^*`4|P+v2a#-9^Td zrr9*64tdV*)vid>;!&{&S0vp92HYo#&&0x9oe~-*Fz8`w?xcG3D(!kdgOxqPMms<9PpF zw7Y;?TMp))PFCSz@>!T_{4BMg7ArA#UOnHzEX{;n%F|+sJ)+XoxY&Dv&3jsuYicu{ z=*-kYTaia-g}6J1sPV{x+vxWr(b=zGzgCO9Sblq(jCKF74u6ip2wQXv)wu)bzdZU? zxK-TgX-e|ZYiljtY2taWFD?FlqUz-~-pz2f&^6?TIFqFJ{HgtCJQ}l@T|8@ow@T<_ zFLuW*-y&j7k8c`h%*J;|2?)TvAWk~FOqR-Qtqb>B?BmEze`WN(vRbL%{ACVQR8+>Y zvR5^O6L9B^GxhT|((BvX&w5V&3{j264jTNH|4fr(Qn5H7{dAt%cz!cb4sTecMtXu( zL#kRa54M^!pMb|jCHv`b8n@nKUyE<31r+11-7~dAtv^{MYQIR!r!Zcig%DLQ)PS7d z3I|hwKb6X7TWpJU?^LY~Tt)b zk@mCW2O`~{i2c3?#4DFw6I0UWG5aA(NM~tgYHCA5d^~?e1Mq;=60TU|4)w3@u;UsF zUu*6iP|nW$%(K?_YH{Ow4yo|;F7Mvx>Q1y=-Z}B}IbVfif1{I$Of@`Km&qO^K$bBGSS?;H zS$WDWk&zCtMz!9AUQWHx>vwLqtE&6_tA3bvg* zK!p9|PxWu8Sg9W_EnmeiRDCl26vI@1Ln&6*rhGZtC8N*$n188R3S(=rPdnrJG@XnG z8YyN0XhztDAGS;b09Fo$$RjIdzV zZ%2Gb23FSV3%_S149>alaF$2|&PzS{BnVBR3+N?h7bwD^=$`c43$53s7iUY8($VLm zMQrHl>3z{ZJFrnjM&c&LJ*LyEbKR%<#d0P!00>c1tM9!BR(k#Y^%aAS3EQo5Sn);t zii@S^7Zy6Yx~{xeLu{ zuVnWDCg&Xv!hua`C!aiZ9@2{*+MyJFn_n=jlAf9QD)us$ZCfGXPBbf*qtjky+Nsv` zL2loLZ=!`bjKD4{N83|Z;Hq=y_{XNEqg<@|V+piYlM5L;k^LU4hw-y}Gl_H{>Im00l&ZWBK)roO`e>Xx_eR4er8 z`?Mo%|NbbNb(;jPDkUmuen2xi07mDoX?`AkV*<5a0>e}Mh1p$ z`QIKF&2h)57dTml^XoFs47Wb}x(J@85-matbb5tzHEGKIUovjs_&iXK6rl5Gds|VF z0Js#Yy!-d;>;rO5de3={O-$^^I^`+@d8lSQYj%1o=lO`i$c4VQ5N6E+$PMjj^I7t$vk<->fA{tp+tz{{)u>E^m*NYesG(*J%U>Jr zZE`wE&AIk6CqQ1k#&GkqbMbl^xqRCrtIBYsPscVI?mgB<<-W8qlkt9!L4nr(SE!`A zcV2qVW#2t$d!{nGa-=_)?3$bU0-N$Kv|4Eo1DZPfs~xld@hzU$Yi6^rYTmPsQva54 z0MuYrpr)ybo~oT~FYHS-%B@9|r7my|Nv<_EHs&7q-FZVWCX;|ckqVcQw$@Y?`AJ@2 z`y^MiEPt1CM~@zrC19(-bbnNLEaS=Ir1-l90#ku8L+XadRa9=C!d{COhuR=0<+J!L zX|Vj(ma>yi&NOET^6@F5iDasJozAtk*Bz`mbi8uJe5cowdn!E2`Do`a8#TvY255}r zysd41G?E4~tC{_c;4@F_=8z(Z+<1%M`r*4Y<%Lm_fmdvEykNb=jf?lr*4XU zZAdS1vENiP6Pq=~eKA&S26VbiA(&maH7la3$JoN+@obe~@u@qsYu7T$?YkIPT4^R5 zyzHF!qs7If#Kw)c_kJ4(yRDHLa;}b(ZciG<4r>k%`}M65hds5n5fDDweJwLDXE_9% z0ehfWKJD5#tdqx7{|-MtKcz1_rwS?bz`NU;w@hPObyjfrSm#v7#!Y}LJa_Zyn{uxj zLvza$`vJb5yV)7_&?>q%q@?6@f|{yy1j!y5x4#n4+ghgo@I-)-kzmCgB*$saq0f&|a$-FSHgYN}1o1HxW|s#b)~z+L~pe5dCZLfOYSO zsIJeE*Ra=H!n0Sjq6$o8vQwRYu1FM3-xhSgGN`{JD>;TrU(KO?yn2~ zBKsOApS$ZfP{q1yVBXT8iclfdjk4FYOoC3eoS4uSKf#(eguuCLeh zoS)57jm5so-SrY{#msTqxtmuV646^oUV1i5J&|W+uN^g-E9P7s!JRDhWN5hcrOkWL zu=)$08U~9)Ma2N&WJ`^qJTUB3i{V{tUs{N3=)Dpg9IR@9Gn12|&_4$~?22l>q{nnq z__Kp_ExH=LYJ=u+o@b)f6Xcucy;RuvCX;rqr3B=hZ*&&jbWI4L+f1lk!_e@7_GQJ? z^FBrHQyLf9pEdK(Eyz7QAsS=5^MHBn{UN*iJ5%Bgxs3;)31+3F9l5en#0+)*e(K6+ zFQt0T_D8u6eLe8f{#&7&=6*Vl;2m>McT)rumDDdEOm1f{vVTs0E#jyru%Y3`7mYq3s#xkf<7n($Joc9|5yd8~~ zow*ZrLzxn%4ph^uyh7Fb(j~7iUhlzSs==1L^<~#LgdMtoKENXFUD!>}tPtZxXV61H zT8p^TPtN9RF905RPv`REZ|wy%+FOuix_;IrtX{iTNpmZ}{_B?`uWo=3oXWlC=hq47B$pt6 zi*ndu!}V=4exhDMv$L)=EPG4(b5>CD^@C6`PG!X0W?!1|S@H*WH2U!&s_CiGa$#20 z>lKjA9~_U9df`08+U+3bd-Y+mMqFK~|N7$%qx>#jH3tDT%pV`q(1=s4jg$62`q1u_ ztp-${QsDFph@vGxkerM{BYp!Iun_7@oC z@MkeL#R8R^T36nj@tkHAQp9Ff8h1N(JT{Yas_avn=(|T_%97Gh5qUKp1$1Xkmq#Vb zb!MBm7qwaaMg@rxm*U}(cZk~Wam(@~E0i9cO#P@NldjIr7{16ajh|aTm<#=y83~5g zb=n3e{NVIpv&n&T$215M@}t?z74=vt@LBD{v}0ZZV!vW`tH)oh-gGP_>se*+-pej^K!4$2h3Wo>!1bc8 zqlWkT9j1C^^fCp{H9R@XfrFcSGfZt!k7o1!w|fFxo}bFKj%qb?sP64sd|nd%HQgxR!O5-wKQIuhfkk zLHTXq=^KDj0J8oN&B!w7v=RtDkrMcIv#CKGXTiJoEPZ_`B6IYp9r!@i#8!58f&8}@ zt-|&V|B88ndXn08Y>z9WYx#jX zf7T}lcfWGzdH)cKT}og~+NNx}B2Mm4^-;IspN~bE@HF zXc04D&Gox??+Q1zKeyBb3@Ep1L9=Aw;0XTy-7vcRJcN@yVa4L(qa8BBR-fPTDBcE% zCt6>R;^jwBo$4N|Zzpd>B&V^#W+CcTs}gW?Lwf}EU)(mz;=aQWH?eDKv1{gg`ZdY5 z0s4c00nRg$p0g;Hp9BwSJp1}3MROIY*1zTz&s|)Vl%~)sKxv>MRhbF= z4)C|yz5Wj>FejchCpB0xu(Q*F#6^1rKAS2~f<|UjaAapD%k59r^BOL4D~=q79g&f% zxI=P~)^;^5t?IVUs}J{90`{#3FWRp6+~Sh2FQsY|K?W}qqg~85qia^v^g?q3UCO3e z3dug-C=hM5-65Y1pDc%UKLOgQ{3How$rvhi=6b0OfFHhKUZ-swo%S3XauhZ>m{4gJ~z<( zY^QuMyUm0sl=sfj3xm<|=xDQxzeh511LFfA6vYG9T4v5%Sua%r>a}OdXR3b#QBk8F z{TeM@ubDoA_qBk|QGKGIv+RZmN{l5tUkE#HZF!PvH2L<}MyH09e;xeR-p=8W|1Ig{ zYt+-Ni(8dy46sOE@`?@Wc$E`4L|Fk-qS=A%q2;nvAvdnUiNR%wOfbJKFIN(HrkNm5 zTQx$6G3aE*G~hp^#T9C?Ef&Ihw-nRV6Z}-;i5tX1ZfRCau8ne*H#!q`ax=fjbFbgd zP9trSsu#_Za6pv>_Pqd!jLu42dF9BsvYk88)}c8)?aTgEytJ_1e?3R81kVUASj0f^ z(p5LYB&>2&t@71VsB5Bx&e;KiFq=du346gC5OEPZz)CA{_En6r*Jio?;drf?56|yG z3`Ds$aje0e=!|t~vv*DZ?AQePG%c$)@KO1BrOI~|r=OgWjIPa$m2i1uy1zLFjqhcs zvaQ)2!3*$n^5$+3+;UL z$t2f=9`K8wUvtu`4Sup7urkEG3w6{uaAJ@RSG zW^g@}!W=guo0LB|awTTiGE7^lH?cn;D5xBbK9==;*t}5SCx4ms0sBiHt0$y?kv_cgMS3d-Lo}<|P{J0H9!DlN7 znT>mGz6vL^M{C7G4q4?q{LM0F2>XM$g0ZPW&j1D#9Nt8e{%%$UX9Y!tZF-~P4FKy( z#Ry^)fu$*VSAK;xBJMnBQJ*IB@WiQ%f!N1dDHlB9{^N@}RBf@0iwG2+w}Mvfi4}7PUnN zvhSy@s}_3UdiZOkJfBAwqK0vml9{X+#v|e~dZGQ*EtDhWnoG;Lg`J?Q`%x`l`M|Na zM(20ToTn?mOORf#C%{33*g5D0u&VZmJsgYv0kD1d!1;)3=Ogr=BsXc}z}WDpw>`HE zt7e%CcV|jJQ^ZQ#J4Hrf8H4NFyM>hT{ z1a(&iKMD)v5eIk#(%J2{wf&__t+`ZK`ncFxfT?IBkA3_0kX_`S=haN2(aXC)3-<>ArVGlxNDtoM`arWT ztkO{|aEsG{`}<5*dN0nM6SnwRlIXo*1j(ASF;IObrg@bK7=y_^k^6)Gde#X8qibHLQ~E!a;0e#yLYE zS^#cG2fk#4fR>}fv`og^-2t8N!gr2Xlf`J${%R{Uro(^%3@j|GAzMSL-p9Bp(2o>Y!i!%=KH{#);! zdwPF)tiD{*eX;~yDI>S`-hp-C^hW>)9yShSp{P{O)WHPI{;a=O>t&?7I)@R|G{4LgppoycSqlkJ?LtC)xDyOaLRUG#_f`VxywL2NnfGexk z#~YVjC-yCTobJywBh7M`maWKcz55fJq65W)uFBQTjeuRX1|lrNWx65djP=s<^rUINJe#2-p?}dBiww6WPW4J73>_#cWgw0{qw_V>lh1cTkt2l) z4XOhG2Mr(rp3&&UL@CIY-LFZ@$`0QnM~;X%GSF@nXT}}Ri~0lWiP!YcCE^qipo8#H z20G9I5BiTHWS&?%H&VRBPEOF*+;g?;%)51HXt#*SW*L_~&`D!JNTDUqwhKmGM6Jw8 zHE~rthiy7sn{ab=?=^M~L5x@Rs>zHR9@HzZ$y5}W5?#BI-{M)#lh47$#l^Ef>$XRW z)2_VIgt{dxc~B$R49zVb>dIIXoI0fok$TE*uv*p1OrEIis=5D?m@PewjX7%PW5IeH z`^UJ?Zr|eD1oNrVF}WNUwWGbXX8xV&etX~ux`8DWpP1C$oSY@*Em+<3e)s9YsGq&P zk;Vp>E?qK)Nf}chAk=vP`{J=?uvlL~TaEsV<-(wnUU0MZMjVRh&V5RW8rBV|cdDH| z2AcJc*qziulUnxXfeVbF!S({XDn| zNqt_i^AiL=#QXazT(5w&tjvGv=>yHiqGyfsB-JwgO-wwmsFp_|W?{HvFsnvNz@s)$ zNE#bEJEgYmcYQO*9i<)j;hb)$jq!rN!LRrHajWAL{Hl(zzB%@L_h+;LKJL|~Xz`VS zh$wy8!pO+=1ANHfiLJSJ&CY{fzb1tx6*Sr#b`6P6`o1e8u^K?xZ3s~~S1}9dv;3YJ zi7X83>*%;8kep0k7~+xCm|Zpn*5En#=0dhzKupdTbazER^y(Bi8@;3KkP zoTuyK@_3q$1sBij7kh6d(3n^UxY;tya^V67tRt^}3%r``T<3rG9eBfDv-iw5S@ta> zeVcsgZ$COdPVxjNP$@sBYOg~bl|eBcQv3cX&-UzR&^i)P0;(G8u4F<@ zdFVMe4m!RHc1f6)kL7QO=^sFIfV+lwi=z&qGp9qr<)u-J%A6?TIs6gSl7P4Kz7#cQ7I;q8q zUqct#&c*d6DLU)u!;+}?j5uZ0=VwNnL^Oz z*bKK`rbhayeKC?A8%ZBQJb+(D33lz}FBzvzK<&_ZRZ{&5wbFEG@eM0rx7k4#KIA%D z(s!{h7O5dv8ajrE9uSK?h+XnJ`t3FlU(gfKKOiSY@`;eGnhl0;XD6Z2Q!ighQv%KD zscpMdjI<9sluM*AL`#IjYKzIpEj`KpE4`9&fI^SO{ zC@2^s<;4njyc8Xr0VyIB`A*oi5bv*o*+%Ft^tPioJ1?Ck{Fs~w$K1@4N2 zt3zA-=GwY0=$`t1cxga@{&j%opSSYtyW~IImuxYo?U7YiS4ULlfokYsfk=`}8D7l* zr{y&|OopEabVO>>iu8=GkYDF5YD(}8Xmds4t0aJ7JLl?UZSDhms8_6b4ITyrfY2l1 z0Nvx^q-dcv3hnsvh;%z>a*iJaNo+Hpx|f%e%UzXOjvUj*CufgmSpzhH_esCl#&;gZ z*RG>7GW*WeuvO&(xtYMhKlRetCUxgAnhW@Vq!6Mk_(erouqavcK9Jf;w#Mp9dLSsCUWB-W z_q-^8?k&iCh%P=r$G+gjMlJtT%ry_RrA2d((;*p(Dt_=yRq z*G+|P&l(m%&N^^dWuWd*mF)k_$wcj8JsR3D8 z3I#BE83a+t{P9@U0NB82UV9Nw0`jL2p#yOC1h6eLpVn%KqNt&}NuVh07h|vBpQvQA zF?Lz#H}8obs@R3hf@coh3J&^v?Yw4);`k!WgiDM_>T=YIFR>O`PPr++ozOVKMYC<; z=)dq;lKS%GY%rF_2qFYMHI+&E&DS8H``jljkoa{P{QD>HeYguZ#KQZn8%?K8_MYY|>0F}C|XZ8JYve;mQxr)DoXSKk$}VTH=<|Mclz zFyM+5mdrCneEE`D$(M>2bju(kQ27qLWlCgvvV*JfdyZ^3Yhgry{`Qyp%G2%z61x?4 ztW^r?!PnR@C$fy?KrShI85?oA{>Wn@$&=2y2zD<69gQ?Da&BN|phUa=n%qSa z5J3B;oe5}(CO73f z?gZYug1T=40YcPmoO>LLsTh3u@78>`;GiG8jMHex71#rY?{06~6|o1|Mtr~zzsG-H z5|I6hEeb^%Kk+wrUgwfk|1h{W;2Uh<(+kld)JJ?dS8w)a-}NtXA5xFn5Fa0p_&rt8 znYiW-`usa}7St3}&zC)(Q1C|)d_Zmp?4{K~%{Oovd0#GvP~Pj17&hY<_e(Ixt89y` zYIT=Fh7e>G>qo-4=-G}{ZvVarS27|Fr$VX0mHgx4;##(-$wpNg?xj$e*UI_DRauM3 zB-axZ!Fp>Y;ue1JCX{QJ@vQxGLGO0q5K(*~vAk(-mkvwV<{%xVVm@UnA6JEx1af+= zlRp`RS!wZIzJ>C>3#r@^1to@ffq* zT@@aRuSUv;Zi|36e!e?v55I|7F?`DM>c0zBQvX!0XuhXeelkf({PU%Me~KHh``@<< z2zYn-fBrPvFdkv}KR^6;|1AN(GLxY3l=gCWo9i;q4rD!k$+fZi%iH+IB+>L&R=zw^ zT|1o<(#^)ZD^WktglYy)IPm{_Z{8|5Hk0y43GTwDHs4sIFuB^>s(gW$!N{I#SRs)7 z%hl~`e(&7c&}Dy9?3qk1*LAKS4(XtMCgl&Xy@0OZ@f;@ES^!$eI$wter;vfV%DGx? z;ajoK+3X+yz%E!4GSGk!XFQRrwd*FE7$fjmXpfAHOh&%}jO5aa8MqsIch=L=Ui_7> z=$-kNBT8i@Ffj~MCUpLzrlybTFYs`ri!Y2za$8pN+oZ`?Umm}? z{^J@FmX#POG@0{B(MUXjNFNCRk@yA>Toz(I^T=rcXz5f$>^@0{ABr!Nqv{3lpfV}NP15H0$%ue_a-kGhu7>>Rtgd*peC#SOBbd1%Y zKt=S7ZMh{+fE2o4DInTY&p8L^G0domdEz{roD^J|0tuK+fBbj?QLYuxe@Xz}?|0^# z7#okay0jbNKnu9>?+UzmlW33>&Z?d32oNJb**Blns{G7r$mc$y!65!%WaFM1DIqfz z!xZl8TuKrAfj}r7I5d!^>4+W)OFmscpAp>~GC~Eaj!n?kgBRE!_VUa*!X(0w$ubI@ zv-MFQ3KVA$oS7VGTn|%~3 zi}6`_LqAa1-CJunu3?$WVyju}0fCAg-D6|E?6q;-rfk zUU@J4(q*K^gvss?=Dl5#m>d~yD_X~C6mZl5Oy{V+J{!s7rf4R=Dc$~%$qi-csd+VP z_xn5RNMsBe=-~YX$O{xk_YKDszeyREP?EdUZi!#)|BgMpk}?RsCQ}o#^Ef+E`ov@D zfS0RpJDQ&RR}0l6o=#q-VknqwV0gIPJKxYw^>81?3sxa5;Nk-boQeYXC0oMQH~j6` zI8k+?_M2Hx+d_=`<1I#$@a7TB98f4ojYhUQP7b$jS$mC;RDadlv?^vXO@ZVX9f`xi z#4ju=Qb(B2bEee=;O7~igx&YUXhxsLm%7Ii&4X6C{8x(h$MT$cTOOTh`F+W;-uINu zmov0S(i-%u+ z(RNdUoPD z0*~XH>(rJx``%J+-4I`9K~usJ*;SVqvmo(nS7q;x!vbNin1+)=DoK0#V$qG$nBa&5A`J5z^2Jk^ubdgc75+^&pbvM4-f%D7(AQ zguim70%U|s;QZTwZXpo7!MWWtA&>NVCswxt@$!AfZ&xEYgyvJo1OX~*lFv@$)rU9# zwW-smVggqByiL^qsA_(Hzu}LR=jYgiW+sjviMy%DOqe{%n-IkiGcIZzd2o;^B@lJa zTi)dXD&0+4BSSNIF%M}3K?-2(c5(5q7oPaOWhX)65`5|JnC3yKl6+$jc#HN|l9Ru* zYD9N!vviq%M3QzOt-2X(Aq)ib!(t%*Mg^Idf-08i`T;2tjB7X|!OMXon*DVX3rhfz zcc80)`0at3WpvOo?2i4~?sp87Po;E}!^z5b zzrU`pufHIv7-_5`KEF*2)(Z*P10T(zvQoJDU$QVUnM9SAZ1|I~TTzUdfPKu%&(ywwq0`HbSzfq`n_-B6$tkETB;m2G zln!jbHtNORIx>4jav;!%NI(=up!J>ab~ymET?xTbZ@_>^$Q1dO+6&E1O{c0-nMk1H zoUS%k{#zqXR1gQRk_-Q#6!n!bhAirnXlvr6gC9KL9BeCU>C0ckc#@UE7$NZdU^*wX zNebG@>VSA&T>*R7%R~{L#}&$xL;=Wa%po^MG||H#K^Z(fRjUyS|5I0BrF|>7xUP79 zd>yIYG?Km0If!X&1bGmFhzG5vDroOLGpLh^r)l3sxb z5F2A;E=6zOclbku|4|<)fsr?aj-1KndzF0!31{<%y`V9y4BZjSD)kX#1Q0&RqC((; z&gmjPJPAqYF=E7&irvC5lt4Ae03XFp^in)>S3NVE1kW61mh+cZjI{fp_Fwz_gy9$8 zu7$!+p7Z7O%gqx_#$R{8N>4jBle#MNL4Dsxc4hCf0k_p};9L}fb&Ye&l; zjM$B_nYEh)ZlchAhXUb>2&T<@(%va2ioLy(Yg(Zi-sF&G$7q4S3XcpfQgefcJH9&Q zu&e6}&7PsD=XPK4i^zc*$B5c5gBo^+pNw-7K?w`_2IYEi`p~<8*>8bJ9grX1*6#gO zE4)lC7yC|DGD&4dajchG`{1luty5Xo^+X2RmCFk*4*21okbE5xA;ClPKvGuz>>)ez^T7Vy0Mr@()GF==9q z`D0Gu`TZ+KB2Jc^4#{`1Rpnv&HQ9HCBtu}j<7@hswo#@@6@eUp=(KaHy)6HBS=l)U?~lY&{8m4Rqt0%{E#pdEp+8AJ4d4z?1?AT=iNC?%k@F{!0!RDfy~=iki_Rt4rJU&w-j^yq&=yzY)4x+dN;u>SGAorKLkgP#h^G5EwxC zDxjII;oP=GLDsV3KS9NBcjk^~%{S5Cmr<}a)<^Mov38w{i`wgyETS#BZu~K0DD{DL z?}hjUxUOt7dZT(y=cZla=QBQiHa_OwskiOC#p{M+K_MCyz=Z zhR>)l(&w-CXa6(Bt{$O@TU}|pdtbr*og)4$Sv?U{f}OH&St+(FZnrBRPR?HIB+whu z$FzyDNBDo1rzeB{e;MAK6wHeHl{k-=F_8!Ni_k_kYPn|8HJ;XukyI&oOa- z78t+UqRZfP%2%e#oG)_K;tx{)8_f^Te?GQ4Z8T6h?SqTgw(p5fma($7YyayE@hS&^ zMgMZXbI!cf|9!drpAlCeBdOiUK4GqdkPfnanwpxN8g~C34-~Qa`rpK+{)@dN9ECd2 z5-tslMSbMyTu7xUA@PW)28%iVIGkLcC?0V$^*6+lyNBGeOR3)=2ehYm9Sx1FXW@bo zKl&Uy2*%fZ0`$$kT%JyJoiPKDUm+c7{;bsV8vGKT}IL_{c{FO1V_ z-Ro33--=Fr6se97bcVDcoPiP$K_q1CmKME`W)ZH=O4cDCsuJ}d=wnD6L=l_C#4HsB z8B^T4^>)MG#DJ7GAuakA5Nn_5yW3igxN#&PLno2-088|20tQH;gACo6KH0Ko}|@)xdjJstxd=SmBCgb z(;R6Fpw99jj~ZKT!KB8p3d<>&nu+f35=tAECDse(@knkDNhWETnz5w+_{+O8bUs8~ zS5U|&Rxbi;#Df)6d~pksQ-{e@2mW-h($dnLbS0AMi7;nicWsmz$d_-_~|H#Nq^Yim3@-7&_G+r&AP?u^AyN-;$RM}Vd zo_kmG;t&@RmKYMFE%ukPMdaEK#ebQ)rBOd|Y_(ObFo$4OwCY43JuS0I1`rJppK72$b3bE*G zfR^aL*l8<)oDr!PhVV#6!LX^MM<-QW&uYWh#LzJzU?d{cklHhupDq3$X>a{tq`vxz zg!r@4r>+f7o&`ZO(g)U!Uv~TC=?Q&>@&N7`qAQ_hjQ%u;qu2Mj*T#s2(Z>}_V^%2u zDiq*a)gpO*0TP)NpeQE!EwPMOUi;$E7~?FQ%CXvjPn8dLu09`izW`oRFqXmOnw= z@}wU6|53BD$gOt&@sJxkGR5G-vFz+~fvALQLs)&k0)6oB(8+hgNjlyqVsznFNT?pB~m(?8Oj6a!}ME zxBYn9#9r@tSD>x0q1+V4A5w%G0U~1{;nQVe-XFu(rAx;poZSQ&<*hK9p_zp7G z*6H>8@}FHew}WRjNlG0;+SRBT_{h4oV1w_ZCnJRn1Y+KtOI(~4cCU-gQ(gLhO^hM` zu2SN%@2(n$v6m?)(i&1SH&H+nG^34Zk>a0)*0&FlH<)qjV5vu>E2Kux1GJ94Ud3UE z+yNwAWPvdFHMZ*KPxaBckfP; ziJXmR!A=epfsu!{^+o+h+{2+NQi$G#m}TV}S{z^#?Jt`ZPtrU+n7%Em@lvz3b3vOd zHYYK&F;%>Q&@IgN9=o_TN`&zkg+&xws_{i1W{pgd%?eG$b#<`2geU!t1 z#hThr;Fap6mBFRmaZ==%%3*L1Vb!diJS=ZYOY2GT*hsz2pf?)%)5Tu{g7e4`kc6da z6iVrv`g$f1IYQYWH}*L#^xYCn>->k5xi(Ee{70|{EeJkEvk&eh*;^ag@E#??5}+X0!j343QDVGgDwuW2SsPc``&i@3Ld3Ix&vgh7r}DpAy$W zS+a7phChXA{4Ar=mC;_1qc8jWuOg`g+gEcddBWBlextPdtk={Afis1xktM|QGq|+J zFq9P!gD4<4*dIfwNCni7J~lY*Vtlj%4Ik#Ta(zCge#q-VE1C_iJHU|!qbWzL8^iF> z*YF!~8zETzO}Zfqvy4LQC1Qqadg?O#cYe%#34TC1FIifgktVt_apsmSTZX9cfuB?E z;euz`LA`HuXMtR<8f|w7P}IhpFHt|GX1_02i4kQW9E7OAn9Jcryaj**X&eb)SKEWR zK-jw8o8%Zk{$GRP810Y3vtY8I{tzouHMr%@MeMq*sHhdFNThsd;mv_>5b=2nf*F%a z=p~Dkrzn~Dd^sUP{u&aAEbhm?^I!Ns#7b-;T@g|v;Gi8a1)P#_UiCNqFNhq7$J#)F zm%GJ!-~>`#t{xuCDI~h5h3o(C`AwU<(Pk9T)kzv4M*QMhJR5`@SF(9-Zk715Kje%R zE%Y|T?8u`o2(f@6QHm%6p;t)E>}Xcrmg2Lt(Ad9(fjkwDGM{n<3tTNp{ z*p#&#awUm%*yCwCl`B1zkfptG)_blj+u>3;T5Bg3isWgYnpDscN&zpAAQ1!{>T+#P zS3M1Wr1Rv=Ef5YPaJ(0A3~$Ptq#9RUr{0&fw9f4D|CdrQHm;;yz53n%?$qkr?}@55 zNB?7@kU+yb)66Yp{N;TNIvq@r4XY?~nSviaSJLb{r=&l1*lNR#o8eyv1dE1_Pycg= zJ|s@dz8Y9(aAqe%BJF?hI)(ZO#dRi*x9WT9s()UpX^08a3OF-}vNIFzlB0JEhFCl{ z0gI&jbJga@k9mnWjC%=WcNyCn4VJ_~iFF%AFZl)o9l&D_=$O~8 zK&tj8utEoLE{y6-?Pg5x#RG2rc`g*>L$cE7H1dd0c+Zb1AUjZ92C{ouuk zA@QUK4ORqOIQ4(7wDPgDd)lx>qMgt~9s;0{XScv&!XwRM=^${vCa1@~Q#Ktoi%syQ zzch#US@#R}6*$9h&OY@w83Cj0ySRev5@7B08uvf*5- zwa7VRZdqIQuu9W8$-bPs>JFdTRscgUr*zlGaTpXhy#{*8n8xrf9hqGs&!A}*vHHx5 zEI12@s66B_bs$bk2}})q)}H_ao9M@XH-&dMBQ+JC?}buDqW1uOcbc~`6zxO0f{J@G z;9NZC9UM*y1^LlSNSUEP)g>sbe~S8|wIv(}aT;=)(nJ4* z+u|;&%5msWR{+1kb)wX}RJU_W1GR4|JD!75*n9P^#S+s={*xS~#{4FbSRm zwIYgN;i3kQ1MgVPD+%`oNTmQ57>kybm&g3V>z0<56p_SA)64n-$0zcUqu?AX+s}H7 zFpGV7yjj&N0M-uh6cY1?;(85BvYf(x$ki9oG?6^^g+Vo09+jS{16#ax^R5Z5>u7mwIrkV zynL$|-Y}-eeP8y(lrSE6zOOa}x89`W=nTo=#vUW(C!87g7&t|2_w2X`OR1s+Xp=Mz;MVPiR#Sj5F!+-rDu*%zIo81-4NKwl21Yl%V0)yAR<^908@E;^q%~+&Ewp zA?1ll9p5E)_CxNIs}CRD#FujOhQ5G5f-cxaGR9 z7lPQPeiB&+`{pRKTw2G9K*qcsJj{$+7U77Itd8}2#Mf4#ETB=EM=JbL5V+j8;~T(& zhffHpTGA8hoY`+AcF^tFC?$9>7li@^O%@3s3I%ddNQ3p$kdTnsp9#Sf3Kqh?g@yU= z8^wPJ@6szKg#%U+N!XHRg~uxS;>5!~K`0$$n11qSU8#mQ7^rJ+tP6Q=1z|owo+~H> zkzr|8)6l#HXAm0b`1i2EeC?nCbqWPTTcyZHk|qiBdNqdch%5&*6NJ!)-W`-b0RjCN zzDQgk9J?Nw>ryy9`qQozaxcn4hb2$~f6c}}T zQ%!WLWAy#PH)#<}+pL^Mf4!jfEO*M=W%6R2k~NX{#*U9VUT3{q;QWahOPVR)AN1Mp ztk(X+m1b1BMfN7{^nOioxux`M%j)Y_znk~!P)`vp^v}5#Z{DXB8_b&!peM<-s>Z*g zTa~vz{93_T%>a2Om62%e7qPwb%>ON*j(YU27OL-MU;WQsqD&hay z%HcE4!}Ys%g$h@=O{$E}-t{!eM;}BL(v@!s?wDEd)ru3^{_k;*;g#9$UF9s})C>m6 zGM_g;u-2u0pL@X2ot0{##xbDXg&{wJ#y?woN-Hk%U4!wfxpjWu>1X%MuTG`Q$(Rn5 z59@I;8|Xigo9k6uF3`4ya^m9o+x?E`m&7j3CNzGZ&^~C9^dvIK_8W(9TY&y?+zcjqA;ZqfflxU|K>vJ0m?#zp+zH`(chK-sTzq7U9;~MHN{mV z!!*Z0y@>hhw>J8-Wsmv`Wj7xw9^g0J_|W<^(~nV>pEEK3Vn>&|)#0USG*M5!c8vMm zm@zC^V7@NDfm1NN;J(;CrpCjg>aWjfZDJ8<3*$HO-b_omy(uf>^cjB6ZGM{DQp9cL z|9vtqli5@`EyXo_7jmQ zjxmL!JnMBEO}>d?bAd;`e@{#jy7U_bMAX6s`;E`qSRQgLPlwru`T|d@#+b(^!`5T} zN!Hz@g?srgxPRWxvwboQ$fFRcO`9th!kSr<$DUn!_S!|ZYU8RY+E3M;Cn!Bz`iu^h zD7Cr)at9AhcKoq59BJcT`ut(Z^~C`KWBS5Cs!uO$N7R}g?;P^ThK9b#H)_)yX&ZnS zk`EyL`OYIvSDPd-(19N<0%MlZs!qL5`S1d1LI(56FP27QVkHt&|DL%H_T@9B)P>1* zmXXOYqhOa2QmR~WKM_uR5{1TB*wTZUq7^}fft)eLD|`)h3@%+nR6N50Pi|CyvU%u6 zXoc2S{`GPTJY^)bKp9ga#`|)qTkjh)+IHQW@27pS+xS+ffmO&>b4{@MJM0ZAI|-TT zU;E)R9WTV=8md2ek$y;2U~(DNmiH~AfTOT}Qj*;>$JUJdverr5K(okD@nsZOCKFtZ zDrY!GTzhvHoLXl27P)^teHGfs!PI(!AW1$-D&wu%kIc8j)+fYyQco3+PF|C=F;NsX zGjtsAknZf~`Wa%&OxnekgF+>EAj{cRc-PZP?Am=#)n0Ziz30PzdVw42?2|r!u}A9P zHI}bfw@|5~I-iQZnCJ1qio*Z$cffyq?!{iM4$1LvJ6;K*;N1`+Zcpxz?rUN+=asmu1u-rqmF#2LPIWxksMBGe>oIVc_xXK8hVvS4~4oq7VJN^^i(vUP14bg zV3OD)e=? zQy#9f>rnYy%nJ7iNTGuRYyL4@`8jU4@!tA=X53Q=DO;K4VXn>`@h7EA*Q|CzVhgd? z!m%`QdioJ&M!E<^33X?>)v{~g9s zqro!KM{~B%7ao2vi<@X&3EW7hwKj;veCg~)XczqD`Bt*QDUa7ruYs?Q_|MYDS zm{VN;T1HE7VLekiB!v37`($M1iEpmyvtCucc2Y-JM_)nEk^X*Mpxz_(KKxbxnC{Ho z@C6yF8$BofoJsV(7$4J4o?D>Ks+a64{EkY(ak@k%SYYb79u1oN#CEzxQ0CEYGWbdm z)YxyDm$l!W#s{y++_Dt+y;)}d!jn@GH0Q`I>RhQnpEU(scW(w?T46PFQt|2aV;>aL z;jv!`cMsTP4F6($FkugUNU+TYJXFDaJBM!6GgIYC`lTAXH@?RDlRacVLC56ThArCb z$D3yH@p0&wH{C~(xC!_|ru&Z7+c&Rlcygmq4%tVB4mk;}=bipEz!@%Jqu|nH{a#_O zgD3uCP@Pnwg;}wrtPb>E!$!fDE*2TxW`7vRx2=%bp)zeXnDT5&AO{CAjjfbbGFAIg zX8dx;IR$>BxJhdVB7Ltz*gv`Va|Mh0hRG}w8zjSHq>iH8o6y%|hc7ImwLNXNHaX=| z-??qn5nmWtrA0w!c7gGv)9eHDW5$-H3ucEO*H6ANPygO3Wx=*Mci)zg)Js}yucX|l z|GvlJI@PH#A}yUI&eOkOtX6OX(~f1^v%QvTlZtMJ%;R#12gan&y^_0eH%euqV!`X* zg1zK$DnbL^w$^@?Xfn(=VEmK6Mdju?O*ej`9Zo3=5t9Y`H5{hh{Eql*ONY+)_k0o@ z$l3}{3>tmmUS9j%3;B_GTAV?{EsCNS4a~_vW?DcjL|8^uwI+BD(g&pVM>e*HhW1aO zM?Q+*`0h4q$owG&fnsj;C-rC3gF@9-6=}1FRiwpF*eXs3^c<9gN7@TQ=zdUT?$T7- z%}STg-Coe=SNvUVKf!%tEGL*MOK!FAG(d6O_Z@jXmlUe)vmiJ;)gQV09(@5>P^Tji z6ovQP4CcHyB_JK;y}m{kVQm^?%>hY%?DYTuG^GkF-nm7TE}GHv{i zy0wEjC3Z8T&nNeJ+eGo+$2t4l%B zoha5-{S6n#nQ}TAU`Lb3vO9xjM|G(x~8y@qI0LdUlZ$=c}-EHyn2(ZAX(_15d`ysO}91o@fzZzQCD&zCR(Dit(Sz z({fj6>kg6f!8kCQW2j$bXi2)TxEuG&Qy!-g{ivG)16jR>8PbtJ00vj;A;@~{{~h{1 zIHP~Z?(p6J)SKtuu)bnSesjNcc!~c}yyVQCzC@uG91F_-iJ~=ZhDl08gyqmXeua#9~GkrkAL>h|Ci=$j>owmdej@}y@oI`Ozu?W z`U<~Bog7xNYhgEq9otoX*b|GSeXP-b^7FKvQ7t#k=jG$bBkSD_7Ec19o%D68carp} zqv0vJYX3DAN@)jt*zR0bqd==9O_iie6{4HPebdrGD&UQQNgV4`OHB5LuYkSl&$}lT zOwW=M>RD}y^-gUwjGOAsBRyyn$3!K&Y@%(K@6?&^h~Io-!?s#%^*QdhX@h*3wu`AbN?^q0DY}+4y(|50`ZjJd8HTs^st z${#olelW;(mFMwbl8r(4rpv$Askj@K@~jenz6GD94yl|>`|{4Gf;B8nxrhjU!mw9z5=uc__!Ba0OL5h>IgDf9y~mLadBOu75kN0rK#GpM**!@h51 z7aZBL$80Mj^VqT}PZLa7-+hz&P9FDknQTXY(f8Oh(h;|5chED`UNyJwF@>pro#bJd z%y$ohxAdRFo#Hr^s9!qjR=* z2#BxJZtF(K&EUy=^;oZHSBE0DUDf2>j&luZ zTyjJL80{aarV7)XTMA|>?qGD1Zw)z)siJr8EuhKlmoeu2Q-0!9bt3VsJndobo&N7S z99)EZfsa{e{walNm>%go&4?yqVYJ9fr7kllP5sS$mS|JHGqUsfjibd|?_*zLcXKIyh`{8}PjK3c%dD9NH^$%Nk2^!G3UlvJOv}RMca=R@x@ngU z3%l})cE9UVGP7AO?v?U-G?#_txfyZM$LsQwWesa~B6?y-%+Acga6P#Ck848GRXc?( zOS3lrwpq{21N!-5PSo9$c;q|!q9n-`%|p&SSPrgFbna%UZ&4`B3n%!S*N^c^y>48# z2cy`CcOIGr@fubHe`s2IAnwXmAcuGu>ltVe_^)S)Rb@bKBaDRw57%2@c@AzQ5 zZlguWbKPe#9TzxQ?1E>shSVDZhMu@>QO>;a6$>TF>S(!P`n88sKfp7RC`4FSk;AD} zJHKjXZ0+OZ*aEz=B1Kfe{P%D9a$I|%D`@Urr!89!u0 z`$ajINHwoWzVoauxF;g4pa{>Ia~>v%DEN+(-55K<>lv9NYQaPMxlVn|ttmkx8{RTZ zK@0Yv6#r~>r^7e;$)7;^U?Vv9Cij={TI+SOb($_Tev=|ke_-A+#XLUyMpt7$JtBoH z;EB!^l(KoSnS_CMK*t{4I=kL4l`goP+JBkzHzWZ$ErpdQi6i=b;E=u`q-g)Zmb$*Tc&>MB%;;IA;d|xBieP zdEZ51(lyD|P?b9X$}MibU3ic=uHqp)wBc9CDNYssp!Z2`*#BwB;Zu4NJ15=#f@dUN ztQ9z64lhQ|Ue^r!>~f>hse=TNYlS525Vt<;g6F$?$*&_Yaoxrk3*b8(vT)~S&a-b) z`Ec_7n7?_mC>76`tM0vZt=hu^B2{7=wCZ;z)NtofileOiP+kl|awynjV zi#tMIx167I{_2pZMiwUe3`3(v#0E*su{-6Y&nV&iByF@-FBeYI%gW?0n#1P{sD0Mn zRwH70#k6hUpR#X@(R!cxO7#1SJHZ_#bgDnHn%r@b+uEN!-Hde5-IPvc;|sNeVs9^G zN=zIpiMbJ4V#&-bB#y(+*zdkGb3swFoMWkxJ_l&*BXS(QAXx#F{vX!>f{ZIea^(ad zYuDbP`|9(>k%M)qZLSJMrw5S;4TICcH0-P`0>wgHVq!&A zRT88v0+N-6Y1fg97y^SbK>&(N}v}9T&2s!lTnY?5J z0i&yK|0fTQy?cvT3eK!GaB3R0-`=GNeDMO&!~#O*p9*&YK!%tWN!|S8kv<1S>%0Ar zh2qo-pFl$wlx6r?b>YK1ktvXFly@+j|25&2in`~;7J0FeFo|y@d0Cy(pR>W`8WJBl zH9vdl58OLM@*PoQQGo^tfFGtbB>+~}0Oeu?0D8l2bhz&!kfPG!HV)+Kl=sPl$c3*^ zYI^*#esbB(dl;MF?)}-3NVP%ee}(#7K;p-OdNuF@*o~Y4)P#@f1ewc5w*jyz`6Q4Q zKl1&9x$=?OPtQqLd+_HauX>l@3SGbRbFj<}7eUA>&}0XtE?j_mL1&>3$T}bsg7EUU zmj}tA3Vbh*4)jq#$OITG0H5x92v$Qx;RY2L0g^l^+vil-&ajY=`1X`n2|LPW&JXtt z1yu2o1}aJ#k^4`(OKCG&F=%xP5X^N%Rsq_>L`#E(mgJyys+xA^A`lW;wD^GHtYnMu zTA)?Daxeaa76?z9x!}r+py#qU2yTqUk<8g(tOkp4e`vT|v)<3^ob%#IDM90;ud8~^ z|DXpEIwnwmryblshIK#}3GmjaU(jG=h-t~QpOA&(GX^x$5q2*^TnB7;GBnwM$Nq7| z18HHS;_JdgLim&1*0AbIho1Y|iE_^~;&=!LuFE*${)`Vs_F*@&azMvJ{Q@R3yv=Wb zEpZ(pjS={okD=8M!abLuUCE`_06@V!u$*vKoXW6YYvG@JijZ^?CTpFc}LuW zu@j)-gdmF%)goJ3009JaJ&}h>dBVpIc-F@E^6#j@rYFZ%;yntif2ij^ShF6xM;G-b zEw~bl18JH9uvVd8jE+JG2EYawTqY(UzzpxS{*_qQM|ATgBUoyKqpAT?0cToeVLCBR zHpq>3DHBpwhcNw_6puN;L8o#uD0-J85KzBBEkrvg>P}kU3G}tmzOTB6YXS-hBozq( zL{M?KT26hLla#c}IvwX>C)CgmM&@WKqgs+zhbTr(ocuY%w#WFv1g35`!?*j7Ra)Y< z!2}qUP}uHzEBCKBgnv-I&7CJK1O2-dR&w|thQGgcG1^YJ)E$>mh6sytX1SF!d9nQj zcE$a&pA~wP{w-ib{Y$Nbe;B?ov*<5t z)eE?HUYvV9S*Y)zegX0W2BXL!QiA}G^k=|D8Xl4d^UOeik>31J_YenqI`?+f$-uZZ z#U@tx&z2MZj~npFB|UQkCfvK2h+mEhWB}pgL756Xe}A&*oH9_aWmTN~gP?g#kDX;+ z=8GENjk?{tNLp+v{D^W5-D!iUqMMFG&t!_&Dx_>L zKujX1>R!fn6Zz~EXJOSeQ#z$uNWwm`)8)4q9HGk^F4;IqxuY*CRE^JLjOi<9m+MyQ zy-c!uC|Sq#he+*{BbXt5r89f}rrY)v;+)x?&*Gpv%xV;*RdeV!#!j$$j(LO4%%HmC zsVw}ZtUrL^UFn(AP3=94GuR13h3Od07n698JV&0%q~TOZ+F<40|0^?_?QkCnEaex@ z;v2PVA54BjkO2ikNz4x#AA*ygs_ca>5xH-8oyP@@SZB9(@PEQ9d)UBO*FJ*IC))l{ zFYMLtae)m9w|?K2GxtndN^%wladgs`p||xq^NzUBevc0yz>)k^F1s^2L^6GDbn@BG zUU`{!)E&F6u!_WYJaWpP&fyD=Q1Gq=En~~f%$eH_;m19@T+@e7lO$rk*9%*SQMcaJ z<|npg{hC`!TxU1xJoEOXX6NixV{#R9q_}MU?^a0eS-3>u-%+=lY@^weDD~=zVmO)JxH+Iwh24e#Ft8vmHiHm z)7MW-*cvwv{`6C}#X=5~EzM-Ru1 z&;!R`ed2s+@^O)CvNs4wcw@DNR;7%ATb7aW%T$ze3tPo^KQTQ2a_q0q@(l-0HOjCO zmU7Z^H@4(AuDaT4Sp;6JyZ)&T$>&G3ITiWBsaF&s1t?Akf}_VPOKL16(58@4%L+Yn?@Y}; zXHRFN1R4fwl_=OU56b9gzkdV#oLiVXWv1(rTzJ#@;Z*xqhlIptV)b!hh?ys7$Pwe7 zUqH)c%pMo4pf#1e_TMA6ou&VSK_apNf9Qqe$`5!S@QZO~*=S%$PI*cjizoeeYwELG zbKXaS1ZS!|M`5|pSIFs-wo=Zz_Zp`# zlDdpiAd1BlnPWt(?VU3cH~U@wQSY;@Uv>K5*1r(|J(2L4O_%eNDME#8)u^-&pQd2z zUoLFf#lSw>`qygDw!TK=`4Y}}oh@NB!^hUnV|R*=gK-Q&4L3Qj^B9pxl8*-D>zSQf zneExLe2w4es3{I<98MJlLnh;6<>otzw|LUVz`n;8iWnK@b%RT=d^A0^F-TT+{I=lj zg=Wkzhc|+DPs^ld*Otz=->52b+lZmq$Ic-Jh2+nwma(^Q1)E*m8S`KJygROz^5J~> zJPOX1*_Ii$K6Fj~Q2W*%iFZHL)7$V`ryavc%+8V{!Q90>!xo%f(UYouM5Yz8JmZo% zIGfpl0~sq5f+Z}N?`zIqSdHmyU>dBdmGo~pTjUUz;^%uM-`BKp^}hBmPSz71-cCS)U}V{pSu7D zyglVbtd75C=Xn=II6+tO!673GP@q?ZfiPPJZGsqZd1>B6^Ftu|8|b10Y3eTli^$4c z11FjQ82XdDCV{hYRS)pJC*aH^(N|P?QD&|ZGBldnzJdS|g23jPm;L~eP6beLiPt{A zJkH2{i1yw>aeE3<1q9oG49n6924xlMUr=i2(>M*Bm=|ool8_kRsTNy5_$*=i)9UPG zUHa%o8fQnHw-b$a|FJl!1;zNMPz5v|G}@MHou%J!yu?K!IMwcq)>${Dz4Qm<4#M(< z=;{Rw(*kai8pO{>Yrdxi5_!Qm9x$&7Fs_dQVGDaBV(2Ag!`F#rcRTQ))r~RD4}A}5 zsP0!ttzkl=!t3Swt;fs#5w54g>+BkhU$>}p#HE*yvU%h2pMU9-GiH+N^`?=NNG@$l z_(#bi>7*NzT5Zr+;1Z=V=X}xeGj^Vo&2yJN#$To`knlP??}e_Rg^}%#kY9oPS{xJ( z7O*gcMsG+zzCHYL-IdjlZPF(yWIV&e%U3omXXYZ6zRGi^pUWExQNwXwT%|YE5ZyG0 zkS_xAuO65bY9O1BD9EycWC-Z6Rs(_Q8E`gqf@I{QnAEZqq4^1g4LpT}eAQiR{H`bd z3!@k2T@wnf;*+^+a$1!FkRFXYwH()Fpw~$VuQ+O zjQIPtwH{!>$Wh2rl=>-(vWe>;R%iKXr2 zbKWP(4SrUyasNNVBl4q)U31p6GtgpDedRmdSMFNX(O16uz~NitwDhvpL6(l*?bO8L zi>GkVc*ODPO)^q5Mc`3&+GSnx7?OHSq%iOI)1XkdkHg}IeMtA~0rf{!c$B3sTn{U( z8s2uh)|?3{qhKgX_xpz=q#~vs6htF`nN^*$?A~2Pjc*3!{NDm7P@OpKOJe=@WiK~A&YYoT=hby${a{pq-K!~eC7lru?Ima zOqF<4Wg4#=_}&Iblg>C*M3a>iD#|D;6QDpGzqBZiFevajD4tONyu(Zy@xo_7?-!@+ z;1AZ;3BJxX5n{(c?9EiPbsKFxhe?T^0^B;TA!5FJ#O@On>o!*~W2jf=(J^1=up%z#|d0Ps7IC5U;nA08ga zQ@_9En17wgf&TvixM3<#KWe`$3cmNWSY`$zz#{}gD8g#G*EYPFXb*G}vhc)~CH|0PEfxqN5m z##eIzR9V3jdxQY>35M7)9BWUeZ*arnLxqz}WnB(muxIa`J%Ej zq?~QYWoTTsh4v=`?IX?dFZ10lBK<&?$d@w9L^r6I;GU@pQH_QFL~b z9(~)hysw$IZ%JFAJKZAHXHFJ=)k%$(t>#rJ@z7+l3i$&Tj>pPWIN5?%qpJ4A-voSJ z`QYB{7n@<&tIy;S@U+zIK`IJ`O>`v$*m&arbOj}GNu*qXVAJ2j$W=*X;yLfj1m2 zQ$bztF%&C6iaJd%D0w;lF&n0zjTYdOnD+CSG7QK z1IbAWa3Pyiz!=|hCT1V%{jQWwViqXuU1^mM9m+|Se%BG_+{?q!*cDCxa?0D;S%)oS zo{XQm?n+wH;+@hI>`qV1M=AU^pL#38O>e3XI%tUUBJ-`#U%pyIk;|8mB$iD|LOver(1IDW;*wFb(5c?(*q(S#Uum$1VAiPF| zmS}2v6_lJ1gl3^tA3ls}Z3@vKCr!JpcFNTVg>Ks(4ma;V-T4(sBHFjlNrL*hk zHUq&&OJAAj4q|{vTzxEP?28K#IvyXTv+cs*XXOrNQZa~iPedXf>1pj^ub*rhHC~v$ zx8n|>o7R$+-NGsW?BrlaU<;_P9@u_=8fa}Y<{ndFz{7XT*aCCXeIOx(U2D_RY|J>8 zTZIDnj`XNb5F3F~B^IFd8<5Fo?)XsKCI4R{PcTuPN8hRITXs{v=+SJ}oRCa8g-B7O z_pD@uWjWy5oEtfV;60ys`iF>6Yc*JrCknR%zI0jMG@{{@NfJb!CMUii$;i+pbtH{X8J z&B^_%VWe+Z=Y^B}A+c$$?!?S`ERQ4}x~{mUZqI|{!|RSx^Oy4USe>PY|BUYGgSMSn={%5XpawJ!b$nnrhh=59 zk^1}lqfiiyw@u%^;SJEJPZCQKDwROQaoBd3Jo&7Z|1rdxtcs*l)* zSA8JU#Jzct(D!3+N*KulLn&8fZhRb9`@vg*UF5eo((9bS$~5=E(J#rs{@wtT|C2&?DM0QVh9};qG;t0yw&fudaP5m z!y2D&J-q2o{z}vgFjo|S&F-nB>|7JIP@=fH`9+r?F*|dx7XV{Zz55l|RTz#=@og?s z`{f~0UKatWru%f@{~e6jE~N`GK&Y^6xtG+VIlBO=Pe_MD{n=SwDT;JB)S445rgqW_ zMDOe6KU;P{yM|@6n$PBva^E9H(o*N}^T^c7NVZO6_o!K-VMa(U!QG+EF4wz%S#h+Q zpM$i6-SFpR`KAs1JLBE*)gT}d1iWfQ;8_l|b-Qwm#B{XpqyDdm>LJ(s{}O|b9dkP} zcbirsj(ssvU*2&Ju>LRci38u+QVU6;RfWSXv(;w8JL0+e{Ts>n1&A|c0K<#v{_i9} zLFB%nkgPIzL3!f8JdF}WiAUP8$QBthF;f05CNJ0lgEt6rUK_I8SLizri1u|4iLeNsPDY} zKc*88isTJ~>Nmv6HX3qdj5ARbMJEQc-B2X2Yt&^I>;G{T4<)V2fs~_CvD*V|kE_=A zH;syc+d)n(mnbc;a=(uL-ouJmSt4*~HA_AJ!uK;F4OP45Zsw4yOsuylw#?BXlvMHD zZsdjWfvvX(TPE|JH~(owR2lM)C&j&GGuu)_fpE%$+ky%|st%Q=t?Ukprn}{lq3m3~ zm@t_F^r!NGPDc?&j)?}u^Yuq%-~GSN1V@*|j5{CsG%}RZQSU^H@G!%`aCy=*Spd^u1QHqX==1 zt@QLSB<=R94Ep?ML*a|{otkxZP4R^)1lXNB6(SD%pw+c0KV`pbP2U6Rs8GMQ|F~?*^(^6$zDeQf_A*Wtj(BoI1`34Gfh6GL z?}uchv}Qe__X3>1G7izy?FC)!=i?bU20O6Y_qV@=SF(jMIk4Eh@RO9m-^Nq7H@lwd z91(? zauQY>;2kG!@dk+pDV8$fJzF9SIdr$ES>sSHxg~%4Qp$ypPouw@Vzcf)q(JTksEd?8 z#U+cluGzv9fExIw5k4}I#R|pDJBA08MX>#IzP*hkZNoE3LlC@cyBM6c;uQyi0sBqY z>$v{njg);_ryC>%wDr z^Xt?Q;M$*woq>cCkQ>#$s516u=@5f94*&#IS;g_KHwRkq?+C79RSgU|ag61~R3_e) z4(-ACdYht-zJy@oJ$7c|8-_3MU0TjCnno&FPyfSc_WHIgY>}EN{dEh}+N)cbe0VPk zuVjELI8n~qx~@H8nwz^oQD6FR|_#UzTs2L#Czg!KujXBS9)WgP=`OQ-=P;tAlPYe1}* z7lws=={p@U^96NYf}n4FImZ8ojDd{KjJ)hh98x%s=haKje<~|C>~_pJJ@Z=nc4R1k z8K0!v`2m?F%4kVX#RI${uY?a)SMrPSj*i*kSslI*DSN#9$^GOn8vVAR=5L7j>EHJW zff=ad{(}vMN!8^`!ap`Y_F#t(S4l9xcvKN4oi3s4`K=TwMbXw#%ZCnU)t(%7KPL2& zc>X(Ei#_7*zxtf>Pg^3ch75!A0zEDiSs4|+zaOY2Ru2qNf|Q5b-a4baS{+Ejbiz!} zBVbb^67VhUQlQ5G7-=$?xm&0NB8;FJ_3eZnL|BV-TNU^OW?*m?$k?fzftL=Zsc4dlLsDP=!(6WR9aIMa@av;puP3s@=tp$>$`KKC- zAblGJRRpcZ-c~c&x@R9u6jR$8d3fOvcdxP%e`b^QYVKI?PUcI%L|lTcf&s;sXTmiE zu>|a^Y;rBq#G&6-EVZ<)Nj6#(FkFU(CS^jY?}(`K#}I(SO}0ymnp?*8c~p~Aek2#ilZCTDVH+RVh#Y-g*TE0R!S^sDq_ zw;nHw^5K431lkJ3(9I8Ee}xb()9)p`+8y;IM#hdry@dHzK&a;h0bE4&AAThVjvdDT zd|2Irq1uS9BZw4(4o$d512RDw<#%7|_=Z8~?#?#@BKH$IXnFmRt~wN#uVj4Kx;bsE zO8mr@Bg0DPLYW&g!Q85B)h-hQZg@;1PA)0!xM1N$Qy=56qPgHD@mz(yhk=EdO@)XTNHKBVW&5S~{)RpU;jY1!9|& z@iB34&ntx7U=CqW_*ZT(4L4)pUHaUPFgPivp{q?A3RF3M`pd=u8ak z$+u(5@UBWXx0*KZACvNv@TXjTcXDh<8S}6`)^U^}xV5z7P_iR&3eCH8C}wVU_O`Zk zvG;2K+d{vbOgpWxTSBGYA_K#bl|Q%2N=xk`3>qZTMf>wIttH}yeJmG@lTPfMe`qgn zSsxS;>9y}s7d|Pz?`t@%9cr{V$Sy0G5cIZ0Ia_LZ;76}?Y)L6vNpwvuuZnL* z`&%j+FxCKM%?@C>GC_+HYt?i77n<`zw*t^Sd+1O16f5DIDM+3|-4dpv3p=m;)dKmR zUZ7eRf#hH5h^?UAsG;zP(>vh38!%RY`V6Q(#31AFln#K%3hEY;ec$wuTWElJeJ#$d zR84~ed~LjzjaL`w8~q9d3dAWjTi0c&)FNJ$6<4L!Z%0ggPdX%&rv5h~Q?;r;-~AGO z@sf+(gFb8TD|7MGzdBenFNN{AE=?tQd+dC?O;~Yrt?ebxgKISoi~7_e9cgRC`#Y=B z8KS*;iOyq&7RGlqd>i|}*WvzFOu%0pmei=7D=Fj?N!H1I+3VbP@6F@24L@wJ$h7E< zODdPgHQzT@%(2f`NGnb0H>>p?c6@JV--Dqjrc^o_>8lW=^Hh!9Hpnq#V zHjAnBE9#zTI9+Fh)ZvXncln=}FDtp{yf4Bz*3aw6JscX^8u=4nx;7a4H14z@o4Ka* z%iX7Cr`N;!X^G~`dYaXb9p!$A`&c^gDE_=0cR_?lH6C^Y+c%kNn1XN>8Dc7$d+fN| z;%%aqmX@XzIoqCaJqvm(8tL=(m+dsQwWE!FkDV*`CSTL=JqiSMvXmC?a%J$o7k+{@ zSnC7C)D2KIV7Yqr39yi9Kx0MnWbZM|5|uK&`&$)%@$Gk*FbK0Q*gr_W*8)X(o)t!v z*~bNZ6NBPp;Rg0{Zw<2+p7$HR%zg!YDwg#;+Q*KL)f$3pphC7rJ|Y_Z@tJYM^c>l> z0KF>xi@Z-4uOy|d;P4k7`Dry2qy@5TB#@)O5?P5nSc}G0A}!1JC=)t0UyKXTHaqwn zl$S2r*RE$m+q`dlH%IyP)$$6un73DyVxkc(F{N7e{N#C*=w zeE3E}mHGo+u2T&QZA;`^i^PA<(XF2r9!e#3W%~RH!;89MB7JZegzNo7Y}*1A_q#Xn zdc|T-vY#ckG|Ruc`t5TaW8H)jJ4T=Ry)zQowTt3Dx~1vRa^p`Ay2`y$$hCPDE^Kc# z28;Br%rwj0aKcXL(sEmA-V>9h-l8(89@Vk|RW1u@VxPx7#lf4SUWOVSxfx<^W<1Ac zeP5=}cZm%6Q|OR#XX_Q*8ZdGl5!p3?0Z8lK6EGr!{76)juNDwsmhg`HAO!LgxEja* z9^4MM{^f@TwE5`BckhZotlY@4O>pjtoQ$=!g9S}gj9~PPS^TfT50X`_Q+oJUTB2g3 zQ-0-zzB94779P#G<(7_KYC_+lIj-A2aP6%gh6Sm~r=ZX;jbsvRB=@KX)9bk$toMI* zR{QqdUTE`CW6W?#+&qZN~plxWP$1zL%poOOV(kb^H zN(Fw7g{XJqa_dfvM%CC zSeXo^raaBWy0Eyqwa-$zy~3Dfb(64Cf4)ORQc zx;YeLRTy4~AoZ4Vn9%r@EzwOD28cB}GW(>rwkk_()RFTO$RX~vWgirY*_B?sdZiBK zimj&|E8oK7<2jWw)zeT65p>k{kzgc4AwUrc%i=(If7-f&28yJKICUU4s^>>P4Bv|RUkvWK=IP5o5 z%wE%6I?nbqT~D9<(Jw^XiaY;?zT#loBB{Qp7-x57uro7QG`=jBYG5>H#S#r!#F_+2 zLKSh>qJKTv67vI(TO`WZ|8`!lq;)S_oi-lv(eTJI`0ODRdQ-$W>!fes>ngqZVgQ%1 zL~mD4Bf5Sy8q0HQ4zC&-iec9=Zx3kZ7lLRC;%|5=x78axut4jtE9qpL4fJ^L=j(EU z-npS}`xr>oD2Y5Jlb{P@V%}oq`}*~3|*@|-3KYx@|(kLM7_BVb~sx+6eNi+A>zEu6L)swyoTFK)mvd+%BEveIV)2da& zd3C42l-*p%%zMhC4|rU?wB^dmWJWWh`VxY;%NsN8{v8$xa3x2ZCBs}1o}9FGJZshP zsHo3)Y)e8@58$}~29Buw;R@IKxd(1&r(AQlv7aXRL6r~;}YO}lg`wv3&m ziqa1EeBZzCPNq^459YkMT$}DK;i!uL99ip;Fe!Vl_KSsfEh_UCjk~PB{dU&LNw`UF z_HMIn1dVnPdz<&ak2t(~jl+{D zvYYPj9P59wjje4Yot7AZ1${G{*LSdrVk8u!0~sEY@vO1*#oKCQP`(;L+Cs#nruw}NReUQC0=qcKRso9K3& zHZq6J_Z!&tfwuDw$X8usWR!_4gb7kAiQ;;ZPVgg~U3$YrO9z2{KL6J4+fi1Q*{?<2EY0Ubp9*n$|daCw$z{{;yX`!@Ml3FxZ$MTH}+C zOzRX{)VcIuIX9P2H+0ioRLwL$p6vV7BC&bVK1?$IjY^p7kEPZyov9d)!Zms6Q#y(= zlb!u}-GnY^V5JSz!}q$20OB8YU2rLu$OJrUk*6b`Kqo! z@3sCsr+ifu35!4|SyNWl|L5VV7_GRQDhOwFgSJtvj!y#; z+V}Q}^ytqgd*MjG$)x#MFp?n_iC)bX5o@ercNFeexnmuGV^Yfc3!hv!6M0M ztNfA>gx_6v$1Ij!^z{35jI8$MTA~?gw?wX`%Z8)pZrYz968R7QKi>!y)1-UX*`=~g-h%Mr4k|j^$@h|hX z#QJ+h42!bfj6AKN&%Zs7?lkc`kInoL*GLp*ITZBDF~GR|_eM&9@ZJF|I|oaP=3JR} zw&-|ICWd?NO7w>Ij-}Tk4fRK=6s&TP-=R!$`HbBwS)J+4hOS@HkV_S}J>2IcF1;`X zb_IW*yqlNc4Yj~67-J=}wtJO@#S*4m{l1y0KHK~=IU|{4F7JL@QpW8THD`^+idFDKh7fd(4tP$<+_>P%h zKP-$*bbd^zvNFeRpJ{&6>`(=~uvX}CuLesp=e%k|W~^L-$IPJ(*WZdKhYuoiHVVS1;=;&G@eQRTEM8Tq;)O3-zcWUh1 zZ2cpu109p#(ZBP3-<}#*?!MFMa{21}4p+kdMM66(yTDoE==d5t8H7vw?nD-vwM46n z_&!sP$=TnEy3=zwaziCnFLC_cvAD*boa9c&2+!S2f|w7#ZwkJUTAfxoT`CF?0ma1bft!^(6zaB1Ih{Tl`n>rfB(8Ppb4HaLt>A-by z=S1L}+5wWbg5I9slTK+M;Y9sbx2J(%-WjCftU=3OztkZOm`D`uDJU#{iO~w-Eyl#{ zIrkdh{8v@^*P9AYbS+KA zu+-LT8lt-MD<$6;ENiC=Z~r?{`L-F?ojP)&zNF)w^41iq&@0J=FpH{;>G_Wk7%_Eq zSpc7)RuvG99oDBnLt2WpNbG`zsTCoJ5`Y-8)gH%gHbD3vmgI!g{up^qGhgRYk}nU;PI@jJU?m;sypEV?G;zN2aJ+D?ckX5|GmMXooV4z}nfn;stjYI>$Idi< z7`_#~1Zz{elxzM{TG60>VY$GCoJk}2}ggthC;=w=mz+M5d4zw>0uHG^&+Un1jkdD$#Odq83xz> zFgeha=dbQxIrcK#rLTfV?)sYvA_-EnnwP6GyZEmt8(^tEyW$vEc6cGCpALUyFgd8F zvk;BQNdLOsA3d7mObw&-lOLiP{mSXW7X3B#r~5nW-SI#25gVLgbTT7eP&|Xl(8`KJ zjStrx^lWzD<>><+&<^a-Owjq98}(1%q?U(*6Xq0+h^Y~gyQca9g)gou^SG}Rlc8@9a?R8igv0V^PT{l%FHs(*2gDobr3GFL8Be<^IeB(f>@CG>)}e4jQJFo6Xn3 z)xk_k{5f9LcIvK(jKtx&uif;HnJgxK0)Mx(UPeANyT*Ys3{h$F87cjh{VAuDQKmuk zQe{`i*G$G&7ZY<-RI<_uZ&vM~87(P%v95IVJu1n!4RjeHCtub{$TGh88vfJKc&_>4 zL%u>6(`~Fu9F$)(Clo?vg!G zEx@hpWLA&RHa9{;sSXATvfJOhasYXbbDQ7*yMTa3nB>71I`3^YxcC>C=_>kcG7uWP zS2;3R5_=Y4-dMr3Q&7qs(mU}}{}S{2Gadd?oK~#tPZtWnt(uc}g)Ei^?eZ_OCtseZ zT(R-^5@2-cgb)Xntd4&uTL}}b7J*XN2LltKF2$#Q#DBAHdh7cMk~>@KO4;$lY<9Ud0?=5m5S(^Fp;SxPVfcq1zD>u5 z->o)R%e9S8EWD38#ZUgCf*0$%MX*t*Y4*J2Uvpav@C~&gb_X8lCm7!z)B6<-^(9K^ z%0KI6*%hjPe;DHb+ltgJHCH8f^6-0+9l#Ka7UrH;NMIX6<@ zf6#S#M-4-A{n3nF}wd(QwMIWo8# z#AFFd!^YMIz}D@+Y2$z|oNkHj`xL5$azO-LpbBpChY{XM00LLbxP0d^reg7wrAvz^ z|6GnM6+bU)%L0Zr+$1H2j?=h8H?lWf<+AhgAHHeH)1x`5B7k&LvmNq+cCHC@fx4iA z2SbE8%`7aeVK^~_f=MuBRqsy<3WZWS@%uWbZDw76KV^vimhR^&BigS^JSoR;Z+6|3 z0Dq8VZstX%w14~MZS1perWE;*>Z6FGd7?MH+c?_Je3#t9!h#0`r#qc;#1~!IZ_?i@ zw>lA@i7%%W@z9K9q{PogHL0jGoOY` zm5jJ*PWqE|oY-{g_yoVn$5cp_$MMwRucJ>w z$4_hxZM0<0+fDfM(WN=)a{hr&)EZ6%>orDEO2YezV4xTM;ugy)?dRYX&v551-9x(pj+I#bGtlM^N z_)0WTX&w|M)P<-NAtX|U$V?HPy{>woR?E8TM9C1uo=mQKmB}|S z1mT;=MtH5dnK4h3mB|XpdS%^sic^Y@^nQnn?u&2%TD@Bs9Ilm5kKqeFTQI}TNlq1& z4#N8eo9!WdaY3pM4h{z1izt#YW4|2IEa&q&d^@i2?HsG8kQZ#-$;i6CHbqy9HijX> zUCj?eLz5AYKkU0pcX+3)Di$Uuw~)|_KR2;ok(4GWGliqn!(*n4?~Pn)c$>tr!GN6; z?|D;d<?ozyV%GRi-6=((XbJz4R%}poc;ysLJ&gj{AVZJAi{8Ri=b+hU_C*xRvIwaPs&6!$gk9!- z^^nvM=zerp*9DEVvw8q@HQ@|hG2I5xZnp1~n<}<=U9{YT$14^iDw))|MhyR|P8p*M zUOW!iCn=xxcDf2|JSjI%`FVJp(bBUINX_X_WOAy2;9|>twZ#-=_OiBbzy-ucH&!-w zOr}^kCEn4~6y1t7y^l#j8UOV4n}dq^FWQo{3ih63xgK3C+Gu}*cCQ={IZ}J|daT;x zS7&3{^gFBK3U3aZpUaZpJ8pAHQq9#FSM=r0?v)fusr#6@n%h)$3ScE^%d~I4ZG%1P zGxTQ5eeguph3FY56#sPTEsO4jkJP`YML=cI{9xktS@{0KmrSi8Uvk@3;!8s>gXQer znibjkLoyo`hep&_?jB!;_xWK56&K^5JXIDqvoSlMb7g)$>c6x7-M$sUvYEcD?Cot)GC-|2pUQEn$UYi$K^B$KE9WK zVJYs^dhJoX8m_lr4~++=i3ewh+s#OyxH;A@xN6+;laDd3&uNO2jv~2SczC~k#U{%X zknnT%^dc(TQkIrZP&MxTxHPEku{aQ4E#f~OnSbNa%Yrg-Yn6$<(*SHN7=6%Y!*;dTe2Z+5oJRqOmeuSPvF_J}M2x!G@>U$?6Jsti9bo2<`k1){nwId7OVji>5wehh-Yz&M+ z%2%%FhP=NLy%4;w0Y=UH{+elYX7=`-JEY%q5SFVKnXp>|+)esN%+dwz4SOfNH%gOI zdZm>SZ>JxgW@yCI3hKTU(rJ04x4l-evXo0>HGXq+j(xv({L)-`rP<2IJEI$>g=?R% zXWqo~Yur$OiU-dpkZhnsq23IX$WIEkAuPF-F~F19vl{LS9JFwkPBE*VTa9;aVKDSY z9W&46XmzFDJ1COf&nB-*9{Nt#MwV@;CS(y!k7`RQB<7AjT9V77J?3CG7y>)!01@CWT9%so+y zC{){mmhPJG+5D{b-m8{-3Hem!Hi)Wo(OwteOMa6t!EBp+j6DEfE-W>FLdOEEkWg>r zMW3!3tGyVWt>!zmD0}MVM+>KC#(y8W$^|zwgA9hNZ?FiO5)0gu&_$wfK`Gu|G=P2Ok{sq zs7VlL8P#B#NDW(U_BvU!q5!F9`loKe@s71w8;4ej-`nwimtx;QrN+1kS(v41PhDw! zn00n&7BQv|ePP2*Ihp%K?2K(RnjWbjr{P&bHY?*k zH*YOBoqiREMPVx~b2z2khBk6b2&^_%c*f zJfY`TGK#VJkS+(bdmnm!hS|46D6pXi|4evqfqQ~+!H?cId{J(ejeDw@#CHlti!AKV zcj|kN7SXsA#@sLka2kMD`IgnPK>I6BkXpa3Qs=4=6Ec2OO60!G)5{m7bUSr&3^h-4lxhBXK^xiS1U`bX+$_uH#eAl zo`EuB_E&}5L(W#`pJl;E+zO&Yw=9P9egozWjej~&LEXP!g+{VHzB4bCArM&EvyVb~ zxuaQaF^f>M#yQ!_oLHF*?=>X(C}mGRz=)wmk0p0RqIuqj;fAFdLueOks;m3MsNiw& zcDFMa-I5&cGI5&pO?C3FUKU0w@XArw-n%V0;ILK z-@~OHLLmf7=iwPSzK!B&hvdvoG-ulNspgN@>!MPnPvRISnDr1yKcP?9qkqKK&>*p`pQeAc~94Q94biQa|l zo9+s%Y;_paG_k2bLq2H28+*@&ap0~8N&$JAs;a6-8j2cO8d3}o!eC3Msh_V<#P zMf}%PmPKSgKOb?VYhsDU3K!8m@>-nQ3Rf@SRU0WFs7km@%a)ov2qKY{77a>$Kj{OY zJV?bmIf8r8qT2WG!shF+`l6Mk^FZ0xO=p z(kI=4*AhZF`lf%y>o-(5^aVbNVp@Af^W)=F-lBD!_sOkOyi9^GzFdWQyh53ms-#+T z>}y~+;gQCSHiEwM56=}GR!6Bd2XT9R7D$Bc`)ijs?0b&H5%Eint@reVSfCiX`r(IAsS)D~WFPmph{%dDWV4QK!n;)R@O@Ze9p@=g&U| zal>z992GB8oaqN7fy`=Vbw>6x?DX~Cq1P;Hqobpta&ooFJ`;Sv_&(Z9{>C$4d`K5q zVHmp;W+A^3!U)s8kY9JSeD+S3Xh|C1@Ut*?%FFnZ+fe8|1*yJD_5#9TC~1$(|2jq5 zFk}n}A_tm%$#?j3s4Z~~n0;c0fvAd)1i18NcgSI!%(5AgMHHhuCcy;j%zq&wI5*0`DkJot^xKlJzm$s7?ZU^7(l?;O~0k_8MM8t)bGnVT`| zEnniN*%x58t-m??m>WGJpa0ek{c6=@FA|~!2yZhG=|L6AC5P=+4GP~u zjD`_w-hORxKircPvs!Ev3MR`#4xPe$Bc{1IOj)g~)6!2tjyvV?P_}F1J>eQK1?%8v za_+~S^nNtGV#Mhqr8;zY9K8i%mKHuUYq(Vz#?)U-Ki02twk{(_{rx=mdy|(B-M@F} z(7k?=cSgarP+!xmbNk)(3ftoWR<)bf#{1z#M-$SERo)LTtcq~s2(+w|08$|FPy=+J zT^{oA;V$mD4PQ6uJ4b;rLPbr63a<(m&~1}_5CeWKpdFhEzj>3M1heT*@9lOlaAB2j z)`r1U9arF)$IyNZG?}FZv~QpOu$~fWKE+j5*&E}Bz#u0`7hHcpcl`Y8`d*w=k*^tN z?t0*hRNUr%Mg%n5@0XB}=u#U)xt0jb0r}c3co3!J4(BA5^sJ({Db<;AmLP;1@H4Ee zn~}9Yhj&m}sE8V-ZB%1I4OOncIV%k=Oj}#ha8EhIIRmvAB%z0ik|b-8_|^5mH)FLF zcl_+J;gdJg+m3T+Id0jlTz{EwLm?U$2Q44zo+b#-IrNG?A5isMV<$wd_~#={)>l{{ z0XQBn!fT4hqOYJuhgcOXJRdAH#uhj19YkpKzL$Ueg{kQt`rbSjgJzo0x~&+{cKiKx zJC_`0x)^D{UrJe)=4SQ7^wVn(X^&@tjbo-(q=0epbGiHaeI4u{-->u1NBc%3=xJELP6Y0Zyu%QQ~xBmO;K z7gN@k?axYRcN<2M8NbD-_lWdDl1Y(ST_3q)a(O|8)~oYX2f8+jfQ@MS`MvMq5QE=S zKO7&fz|0F3OtwZc;4;}-)G$AhRoKn@0@H~|CP4T)Hlktd1FR?#>aBLu<#ZOF*100Q z+Ggfm-HI}{Z|WhzGQ4f%!W%uu{S-i;8bd4I3|q6m>PfN40 zZ(HR}q8Z`fnd5%XyhI;!pq#-HS+AwlRU=k#Eh?fC{T4C^Y9d%`7|QESQnduNitMzC zo*Le?yq$0M*#T<2X}j48ZEr!$=@ztTQb&SXg9u&+Qn6x$gn0YsaHoE5IKe3r1|V#l zo+Iawk&)>G+*yseHG-IN`}F=winYPUk%_)Pcno_P$JlTtN$V+{=YwLmOh3?KH*bES zh{+L}J62cT*;^1#8k3J6J-U7GUVy=76fCr8ps1~mJz>)oZgU*Dt*HcNn~Ix&+Zv72 z0)1uw6E+UnANQ3y=_^Mjqd(`f<%ki%H#I7m6KD9d`igi51y)FT+`etLd>L~p*AB7L zkD2O(u*WYhd-5c(Tk#l>CNAMMJB{Q|g1Mg7F#$XP3Vc??@^GMiMw`9n@V+&sd{Ji_ z(8VUq>jtD;ab3|mS7|C+0gx<2w~|yeJ6c|!-8hY@!ax?BfLLS#Kb^(+x1!zRr^Lto zIX@g!+~O5w9PW6WSz_wkOC$i)V(<)^UAyZ8b7$GJPBcezyKL*PcPAw;x=sqQMzkY0uyw5au`=EH$Aa+`odoi zaIuug%^R9`r{I(@!*m#$jp^ELhaFC0?l7J%U#c3WyFP0zrehIrkICOHrNt?mY*}@9 zF{}Jv+z;;pUZCs_DB*Ynm;*00;4NknbyrTmalNP4s~UWGY$A0$*(u*gTI0`&H5in; zoI=%La+n^gT$`0nvc;8w$1a`El5I?0c->srBeD82@hYw&IFUZRN2hxXV-g zpGz%eAZOdhb>S_Vw@8lt(X4;&oh4u6R;QGs$$F$mDcWFIjXRMvp^nBq1?~rgMtqy$ zB8s{-_vuBXD-fYdhr0CE{;_gNZ-B%P3Hn6wO)5Eh6u>Y#e$i;6U;6yi;P&0SD~exI zC>uWAe;zvFJyRo{b;O+2s;YVMw!{sJ3wKbh-h0Xr!H7lx?_aa7e86I}S{*{xu3c*f z7MuyPiml6Yv?a$A2y?o^#Y_~cFx+k_)YDvN%J>ak-tk-SC4g*(YB1|O^vY&i1-lj`+PpFZ8W`2Zzy+df4#wR+=$ zA}$b0`Q;*y=$kayw*DjOr}DlQDn77trxZS^8fU&f;bzC1!5O8msh$^Bj z8TO3`mTn!kk)pih7aaho;i|4oP%b{BpVM%xKlO!!XghA)fB(Dm&CjvEb5MI{qUZuj zEypbO-5ttB>C4Eb6;>N4lsJW$LsZFk9Ev~~gBmOQP| zHSw2m22Oj7%)CYd0SJt#9Rojp{J4GV7I!&tIQh?ePt4{R7U+5lNM8|>O*E>iom*9R z_i+Xuw&dxJ6OlWB+ef%gCm`)fg%k$u+()t82w12cG@&S0#&Nr0jY{$sz#{Z6x$SY+ z3ha2FV6W$IJw0A3xJsmNA@}>{Ib`VqR&4NoLobq?Ao@MJSiUX>lb+1p{Bp)OeN_<2JRiH~07(F>7-yGTV%gK8 zaenp3hFDf(DsD1Htjm$}VeiycFjaU5l>yuJ-Yl-o*U!o9mqfZ<$68%Gr<9qbXp9eW zAB~swaa!@F5+QjEr{n;(0)O>o=Yq;pF{s{(i|)D~uedVM@CTF>90Q@7MZ-vQ3*L#N zMs(zktX+U!zBJRmG_W$Y$d@zDTFrN7@)10)F6_@4SOHj_*dlgr1JtuZ_eDe8Xt3zRhntxI-P!n?s69lY39?myL0y}A$7H+ zjHHMuMN)QAO>Ngm`gq!Q^f;bwu2$0<%E=cbFb0KZE$|ab3AVz76Azune%-J1ycqU( zdb8%paXfY&7KzS9lu_YtLiUqra*40c$vB!O*@V8&)@Ul@7*~~)c(Mb@w|4jmCx?WSdG~xnsUl8k&6%@zfsUMR5Poe-OZ>+ zle`fu)9cCRcl$YMmr&I00QxF_$j=hdt#EyiBDrXYdBnoFMw)d2uu?t^Tv?K9!#+jv zLQ!2SE5?PR5C~90d*<^62%n-HybK17Yi(vu&YIlPWxDc9+S2ERT-vxp{55IUljIwO zoayM@<#*K?^{4JVqvIxQH}mGsSuyp z!|!bGnuMaVzklHb`2}S_Y%gUj`DXa=u6P_wOx>WKh#~=y!oPxi!GZaYD|(S^T8OO0 zyUN!ljoJP$rglR3jaQx=REoRJg=Uq)<9n?Kl#c^&MpI>Mg5qs@oSEhRs6*6gPEdwgY6n{@81 z559rlsJCWu$}NFaz$ah2|yu41d|911FGCm1el$NMtBz7;*sfWmpyPo;obpQ0gSU z^k$0OR$3x7-fq!mJ93q+o^6b`#*C-wxG<7O_w=aOWz?X31|Ap=O9ZXmOv^bFfSxiG zOs2WbCXsv(Jy>a>t*53aH-(IIN1hKak$KP=fb@~6qG;iQ;+2nHN2Q7KD-wR*sQCLm z+Gc7_K`I$7#{$}O4m24jIp~$cv-4sdq8Y}Dz70NQyD~hnF9th@BDq;TK4vNU;c8A! zIrvo`plh(MHvRGC9q=mCw!Cc1oW5OXi|lTnp}X!5I(ZWEfZ5p*x4oWzi>Vd}OvfCQ z{9Z5ayIAgaz-BJ@NNa7;n7is5(C0JD0`{LNABR`Q>2kb^XU{#K#4MVSWnW^)5oR<(grXbwid^JExJQ`4vj{%XbkTj z_yPY9P=D;v``+ZZf;fNwn*IZ)aj=TmM3?kXmHyT(d46G%J<(EKbW++oV+E_|ulAO- z`9PzRb<_}s#jG3M9?fShqVoMdL-+ay>Qf&O5>YywPN|nFER#h6W&&lk__wi*@jPzFz;X2(eWWI!#PI0 zRLxY_#&T*Pd?O{4ZJGKlvB|YwQMZKh3$uIWik=<|d2G>b`+-Dyo>;~W>md-31PYJE z`2b@wQ;{`yHHPPQ?Kq?yB;k^SN`Y-u8vtnoo_(fGw>(snncBrG{q4^dCcJqRC*?ld zchTD66?VcZqoC4N{u5N&%{2bo<=g4H<_eVVM#b-I3G&JR6zh06Po7um<6F~JCZCL6 z?DRn!l(H-i-!EIu;Uo4~*D7BFa{LC6G7^BR%849UOoEqO4$@u)+rK0qeH{%J4Z7@f z&s~yyo1NPHM}ZAg4lX+Ljoj?-OHQ*WzZ15&C8@Mp`EZkOtIqFJy#w!c4rUgrWtG`~ z_*(dV@`O%I!faULRc(0*wmL4lYa054prcpSyp2z#2a zLQI&~QB`(}#qli{PEmR_Hx?SR=tQd#CxdU-UFC8`x4Ye4qwaZvbf(9*r0bNvSD*c8 zraNA=Bkn!FonGZ957Yahz8#}YGQ0w)6e#Px88_jEVdIj-ub0z{TB8=-6a;n>>4EVo zpa`Q5buKah@8y1uk^^H&i9Sb%q?2S{BYCZf$9XDf?xMS;uRHBFUaD>@>!l*(;r{co zqearUw$(9NLQaC$LMD?LRvDU$2rhqr=At1=3shQr-W)7T|4DsD_tC;fTY2Vf`WM1L zE_KcIDSf-W_vUq$`#+4%a|cAnI?ag=cp1LM;Dn0Xw|| zAxjS3Q9db>qCPSa<}~UyUF|-*+Xj=NPO4oOP~EZP^-iv15%qjK-Ye(x#Cf>i+$`;2 z_Uktx2Z08#gBt%9vV?>7PqgOCYG%aavSw7W67q``Q^dQfn3BXQb-h6&%1A4^ZJ3>m6E#Z6Qr zj~g3K=v9i$=f6wB_ht56m3-zPjSjM4!4th7Dvr`Gp$48lA9$*rbFR$JCJ6&>iM0kvPiKQT@o!` zym;CsEA3Te$B-gKO1x(}Z*QubR#(W$9D3d|w< z7Q;kC)});Jj1NPB4~NJ5<~5DY-bob3>JLOplc0B-;pkB=TO^&&nD&ZsmURD6K=ycW z(ktGUU%F%>MAY9e%jlfaGT)!$Q76?31XQV&=VdpAsZgJ`nOM*d;LJE?t)}TGmR0&B z)BCqm%9ZFPXU#umlxcx@s8>u!I*JQaI*m<2VMZuqpn=dhZ-3_^vu&wk`uDE7m3b_M zwND!AjRtP<`xVWOoJ=>-K7TCy(GnkrbhD{#fd!2t4Zm3>^cp@r91+9iN~)|`uP&f= z$SL3(f7AZzmTKLt`A%K(*TU(p7rnT?MI>V}?SNuH0OKNuJO0+AvGZdwi?ZL*?Y%7& zRH}CiIchJo?JP}VJ4x$v4ZEof2%;wfv+MG5WH_2X^9xgh^2_4fxm%mdH?CODX`=mD zksy18g-+vvN)sK~#V^W^JSkdZaH%#%sC7Qz#K5Yhc6u31ETmwxKD&&O#}7!`eCs#y z$@@K>`DJGT28b0iddL42I!&=Zf6v)sF=!KsTa5N8Zau0$Sek7z(!eQoLN6Amf?8s; zI?BJ+t;eZC#{zvOv#yT6nWvi`bTW>*t>o%PHD)>07$A={M8I;%*BsQdZWX>1@xod+pZflajn1=iyt6XDCK?;olSd^x!?scX;P&(~Vi z@30sjDR*-#%l_+d10M)fxZ`+da>o@}fvx_Lnzad!DE6c4nLau4H635({$0+tXU}a% z1H~8rq7%;PQONqE6Ygnwb)=x>$_HUCQ36Llt?)c+v0{S=AND*22& z%kRcpN33d<8j_ZyE0lT`yJ zi=$j?H+)kvUB2H-BDXE!YFtr`qCfAwuX|{}Z{PbgKPhzRnDtD|jV+5k4WAKxxw9zo-iJjgO(Qk-WuvhUNfR>~W$c27 zIq0YUqePOOiJRQ$CJ*wZsgD%Rm>pj`92;b^y!CT52L@+{Bsf+1vi?Mgv5Msk^_LN4svvtOzPC{)DO zyqs8-o1wdz?8-s>9R3=6T;KF##xdi`XYQ`^*}D7@1mvXg@8raF5N2ikF&uc6IGhYj z{5BgtSaPT*jkz>#*`27K+{1P=Vu4;|rK;__*4%!!b=oQIRAJaY!Dz>usg}ww3)|;E z9;}-=a@tg@CiPHGM1O9O_M61R56pves*=o%FD6|1x{S5_prD)gsg+4AEDaJ?Cp^(Q zRh563P>yLw_OIeJmO4AW$hw=Ac*-#KC@$yb_6Px9KdAKnT}Hf>k4YufKHjkIQs4Gy9$2Rccmqp~33*p_-y^LLNGbnTnXwil(OTjX9CPqWr;kb*ds#la zynThCo{;isRAt9#Xs76xax5x6MB0F)?M~g>psF%*B)cZ*LjRwP*3a|I@HUeGAT|py zC}6jDt!KIHkOWF>?o!XJ6l`KtFJpJvSCa72Oii6Y?Gzp!obdglO`7EEva()qf7|x) znW1_0P>MzHHR<&Ws17<#>aP>u;!DQHH2j>VN%M34HL1_t_iRi!y5Wu!tc+*1K6A;#^EHqyOVYKoWeTy(ww+**a-x{KHp@*4KXl`%h zhMC{}sw1`g(`T#ksW!)D!v=Kfor~X67=bcaWD<-XPjxaAgwbc&v4Bek# z>!5S3B(1^eglWgS7h=uM*VPV1aH%u7Wo91E)CCVbO; zf_rG3D;2~9m)v0&Z>dUc)@fdEcErBz2dm=BUk4Q;et)Sc>y>?-$b8TdzFnwdR*) zrF5hJV8dmf+EjIZ=%-y+sIhyCSTXX2K`Cm67LSL6tCVtM)UZ&|60hYjnXW^ zS6$s)&;ze{F~ixt=hnjK`s?B8$_5{fTT;YimTvEYL06esyeK;G0>gE?tKbHZIdw%P zhdNzrejV-*vxZ@Zre;9J%Wo@=dduROYcS9KE=|9NuK!F!qUipfL%f~(Tl1QnI3tV7 zcGm6;kNk7>0;cQzIsWZ5)HqcC81WqMg6U0$$E#hDGK<88^FqsXs1dQ@cX zHJeOHAFq06PsVG4rP-I;L)YeLM{>MY9thS1>sI-O_MN%(PkdQ>-J0KI{FqiG_~_=4 z+N>YhH$1qmIXhjtbz(eev(t#Ecj7VCzpq^sPd@tcnezG6&#Mo$WV3tBY*S>|xyLIt z-cTh`x9O3CK^SwvTv?iWR&All&CnMa?PF?FKT=U_y&@JV=t(6VeT&9O^CV&uPXMZDfJ|@jMB4xW8o#{0{ znep>{qYTj&35$RX2e$Hvw~MPjtm)roE9q9KY7o|FR${lv|JN)_`#s5&d}@dv5KZ|57_&r-+IiKUbqk<>UZE7lHSpQJ9_Ml zyPj|Q6N3E>yiB};BJMvmt4j>vGy!8KMvRtm*rw|*PHqW_0{;_q{#O6sQx;{?-#p=% z{qVNa{0BN4a+Q?T$AEd0pZ}B4j#8WaxbIjv7XM%TkPSjqSvfgE+zSW_q5|-RXf;VM z*TD%^d_M=X&c9`vRH(zJM47Vv%yPdLRNr~sp_D(Hwt&GjKR2ygdc6W3uDAM=Im??KNv>?Yzt zoh24z2c4&^^`M7(C$X?*DDt|VAu9U}UelA@0l;hNrnR!Fm;yKYy8x2*Us7~(Ibdqr zgDW6Ud&sY+BX;9Y{J)#GEENXO!gW z;(y!(*?IosW(m}c<X!fF#y{ z9=2hwXyIy=muSTQS0Hx9z}3R}*%2;o{gj1#gTj*t^!2$#v!<$2$T6Sd2Uge6ABxU9 zz+4^+3l9)@LC#*@zGGoAP~Y&VD}(#A7z*1D2g`gvGTG>VT)Dp7sJk=(LLXD0$J)h& zBLtp_MwADgoKu(I2~IhJ=t30jk!JNDxrw_KgVzr}w9XkRoIeYP3cvL^ZGZB^4Jw}f zPCMlOhZIA{t~JDT$`nvJ2xrmc!8c_pz$l2u*=kp#NZ2|G1I-Jc9C=LRWuzpn&1F)L zI5c67Q!MBuyWZ@p1y``uzCjN*aL2-3^sfyG5M{OctQ+}zwOM_lvPu3Oh_C#sR}WDk0q5|rQ8;j{H=`e|_y zt~!L2dge^%phGS(*J-xQRRqu8koz8it10f)20^hV4<~lKwm|EcFv#z-iKDrtlK}QC z5$~g=nTW39$DW^ZG-rtjbD(+ zjFMs=rT--6_?zhZ?v%jEb& zkf%jMr$h*X;1$<`#|LMWNrCxI3wzz&Iv%a-T_=MgIZoWagaAzr!v4xA>F)!W^qZoW zs|mZ^RbKS83(vI28Vct$&~cYPb*cfg!xSzIlE*>ue}UNe@(jPr_oAh@@87S72I02d z9&nC9*^{TiZG7|n&V}z%{-*Ii>niu;`D792OjU%aJ7$E#iErWle>@{adY%>h`nEL% z$%jL(9v;o%k^KwG2rwES<8Jh9K5+H8Bv*G#&Tw&8sIH)Fy8jOn@!!w38ht{etY)U! zIowk8|JVyN`PJ7@Xm&7#`yr?JRL2c@Sy}4n=M`^!TP^qgbc=%P2ZtO9`+ckEsXhPD z2mGT!+&JU_t;(k9S=`?;dODg-6Pwb1qglY@I|wzh+MJSJShc6ZC!!Zxko?H~{^Jwg zJ*sD$g#U9iJ)IiEsPV6-V~cHtI3Yx18wk41{obkAu<)28^N{Ww{om^ayq)Lu=SDXX zAzGq-VF7V}n3|e8zm<34UmB?(lR!4d%N?iqxcKXbEO9|W4b`?sOS9+ZCJP5VH=R|+ zB7Y9|SU7_gUR}CS2TN{IN9+1;*SddW`TGu}WVKaC%>Ooi_wHREaz=H`wZ3(C6ZsOpHq`5(nMB1Sn)~$PjLBXM!TmNJz8^R2NV^25T zSe8XP(XcWjI+lNFWnzR?fWKN&r0eW-X?eLO4oRGRxxU3)k(rTm&7=L#2^{l8D)Ypr z@!{N&Ld6VBMZ@(X&3pOWd$IpvIK$`vP3<@# z6ru=4GSuryW4OcH-;7^zd*<&Rb;@_@4$!KNl3uC-uQ%9o6FSIoAGQ>t!GBe9XX((I zD0XSDOBjXz>ugWj66?7(*BJ{^JJ1N}BGa-}XSY&lB~m;hxKiNHK8BIWT38zLj?t38 z1Z=uw31=6CD6oF(U|%ACO;$2{KRhmoZWCFnX+vrlf5FuGnCR$B$f!ntIkQMOza)06 z)oixPQ}7=nI5Zx3rK*kZzL2DsOD7WbKkJ$D467N=!G>$@!b!QH2m%uO7 z>!U^=Xk0npg3S&>0N4=G_hO{JlYplNc6%+Rpv1y7a@~5$rgNaxo7TkM z!$KIxR_bbYC5ChGc=kSfwUa@7w1gvOpH!sz;Rf-l-4V`@i9(4GW{CP}Et}qtJHnIv z6!*un^71rJJR4SVC-vpam&b_hj2yfMLo4D8i+uZxPzcp;|Q5Rj=r z|MEzMa)wgap7!?bv7wgij8`#BV@-&TgBVtG-(OD+DR(&4HSVN=d7Ut8G`eu%5~g&* z!6X(m6pbuPy_IZY!c&p)d5;IcC(F>%chR5`f|e9r$2O3bhVy~>Zh-)UyOQ~7O znn@Qw1x10#)ao>lk^}$i@!a-VCE{BV6L>ZeC%6gT^i6t6_CdTC%PfqhtgXZYO@{8U zA7xk#G&h2%FEddl5JiXybA~KoKyK#KekuvM--EJ?HTv_@4b^O`l)vRqG?@`2KS6lc z#~xcOQyZhOi9ANa;U_{6Fp>sP7C(G6}4*+mB6I50a8S z(M#f?LAy${0gxrDmU8f+{Z66oA_7ax+#w+_VnHJdnBl%8c^L(EJJFf+La$s8ZiB)# zVae=CLaj$2C2^g2HR56R!lbwke=i7w_wm6-H8!8P=ibZx4}-*{s&vH<;#9Lze{R%t zw;&XZLM$;u_CSj+oE@@?5&5)}9n{V9)-V+(WxI)qiBoTsH*p?cSg0d}WSUepm?Y?+VST%6F4mry{){# zP03$%h_)RKSm32u;ygU^kZY=9n`6llce%ih-q&DPP+n6xIHZvSHbATc<>Tt#H*Xbxt%6{VSSu6t@OGCmzhq4_6J{prAs7t&BL9pQ4PoRGeWr2x zrLVZN4_M+-HrdQT9I}^ttVWsABShRztHG%mwa|sl3tk8EVPHSnuQ`|qwRYKJ>^)t} zY#E$wbsAcFF&M-9kwluO5UbbX%emxuHIpj~K~xiK2Gi;ZF^QqsA)oc1`s{ui+7~t? zKn+;Mse)$p2%x~t*x)QACo;}dYG!5*o0E*^ORrax^h3;|88Z`k5eJ_c7I&BtZ*0U` zZI(BSOFgl11*fD9dMl$~G9_O!d)E1f3{g6Fl(>&t!JNgk<>j+rAqySLJ5q~cIw3G! zRsX>RfE|o(6Og=`f;EV;o7j?LEtRRVaVZPokm!XWOBMbv0f`}$*m5`tVeOJ+H`c2{ z>H(Zdd$San<5eg)XLHzn^0Xs># znCQuG9BP%hJYJ`zKm&6VLtJv`CYuogpfB|()PHvV-D#g(=U+B*)g&|niQHszchh+o zrWPFu1ezD{*%1yQ+@vP=H)PtlN7>2A>ARMWPO$5A-)?anY$(1;KYgmj6+b3)#oG5l zTi46RSq@8oX#Hj_T%J7weGY;UyByD`nj0i!TE|RC`)fT6+0;*-JQ?2b<;xc$)yJw) zBWVwEd!ynFXKVG*xqv_!1XGKPfp<3e0y!_c*$bPuxG;Iu_yL4o-r+N44 zWxa59Ly~$e^77aB$bk?$G)fG=@hiaK1;zusv3@Xj5g{*I`6%&nkdQb_!#a~1UYGb? zf0*SrWQ)7nUE}6+R%d@)3@>RYwJ>?+kVR88@$l%gOT=MN6f*DeaC=ul8*UWJ@~@%o zHIImcAtB!({;9&uo~Yq~2obiY@vH=Vn*hgp6pr9DC^!W|6zeF0E5%2);k55?n#F+@F*yUbCLUNI58ympS>WJQ^tMHR z(_=gfu*_*v?Z12Yxk|$1d}rE$#!0lB|5cOAe&&+$QzVxwSw+<$y2E~`gN8X8z&SVS zbA6ZbfE>HN$B?A1|9qKUTwD*UaoOD5r}H$Js}=cvwIc>Zko)^1QAN}{wsh4tWgI7% z8XJRzLyks#0cXKm-ZVCgSGkRPaTyd7-)yo+pRXUAnQg-3CBHg9Q#k*Clkk}l<^~JMB^Zi$TAH@=(=pdsE4>`VJmAuOv`@{J~TCwM8UNJ^MiFrq5+xbD3^C+Gqd zr32||G*DI#?473JOg_eREbrWX6m%lqcKQlzhtoD5zY|~yu%=AWcVEp)RvL@#Wu=9#oR`hV- zAtUMraJD)$Tfmw83@ZP5EV%NSpiSqIkg!X8l41-#d;y%0dS7{GV+nN&J?Zaks3o_2 z;REg5q3hKS(*oCPtmbSZ+OjaZ_@Y4vwFnZVRiyLnAtL-Wc&lh{U$iXGGsfZ2$1Ak&{4RAcH6s2$U!W=}JFb z%6NyO60z6d{6HIR``zQ}@=d^_rQHqJIC@p=SKF1S@#PL9ZLGg@5YLQTC(ktRRd;sI zGZ)gm-fX*=mYkgYG<*}q{q=$s7v269(m1`;@K%V|IZC!vRS2-9nZ={g5%KT;jLXEy zdP1(680C3xtVllPn?x#wJ9i#553N%Aa73K?vT_SU+@^D4T6)(X*Iy7;!>o${&(hfg ziztx?|7=vS>4|ci#)(&oSw7GGPuQ7me1*N92`_(g-VwJoB&;ZMeh!*F(lKIB5jND+6yJ}DHo_cO%UG697l^xlHsAR@9N*vV+oTn-D9UIID47PccI`= z4dBrj9ddID^MD=OEM^GN_?;WCbCO{yQ{zZyDso1$Um(u;EnMIAq4vTfQ{$dWaYtMf zT`bqUdQ5skxO1-Vmft<)tErc^938f~iAu{1d3;gBUc1Jl!`UgCNu0#*nPGVHtIrWr z98)tjy3FA|_CVY`@&c6}&nU^?)G%zkFU{3Fi38_W`<;2j-$5Ehn#fE_k2{O(5&9U1_{TWMiK z`}_H=`5l0YnCKBQDkRj9!IA7vP8Bi?!K>dpm_!|hSLSohLKaYmC4|DK`{ysH2^ht{ zwKP&N8-fIb6>paD?-wSOMu65?ZaHEP&Y6Jb4Zd76s(f~`a9$5K5U8U2+Y1ZNI(Ba$ z-4|4%b?{+=R(@}a_OqQ80-gu-oPWGS1{%z1f*ijIohdQ4Q{O8*J#Iq5c^~<$yNx;K zdyvC_jyQi3loY*-B@scp;VN3>!C-7ru}L1)Slxot7YOSlntia$`gk!z@B1^QE8ooB zQB-$1%DcrLkiPkP&*72n1fMG#GxHGm=p1&Zu|H&5KS|h5XZ6R?5-c!79G0N z5A{`KbZGrO0-L_+TsD^-+V%*}P@{w)>vBUe^iq&?21CRji;cb13`}X|HJf++{JM~dy^=$nyYTb>K@j|(G$OKX@Z%mi Zzts7;g>7Y6=XdhoaxzNCQl-v${4WAeVqX9N literal 0 HcmV?d00001 diff --git a/setup_env.py b/setup_env.py index 1b245544f..12622d201 100644 --- a/setup_env.py +++ b/setup_env.py @@ -56,6 +56,12 @@ "tiiuae/Falcon-E-1B-Base": { "model_name": "Falcon-E-1B-Base", }, + "microsoft/bitnet-embedding-0.6b": { + "model_name": "bitnet-embedding-0.6b", + }, + "microsoft/bitnet-embedding-270m": { + "model_name": "bitnet-embedding-270m", + }, } SUPPORTED_QUANT_TYPES = {