diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/README.md b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b840ba4076f7e07a506ec926a61acb4488cd5701 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/README.md @@ -0,0 +1,82 @@ +--- +library_name: transformers +license: other +base_model: /data/jrlin/HyperScale/.cache/model_specs/qwen3_hyp/qwen3/base/std/p686m +tags: +- llama-factory +- full +- generated_from_trainer +datasets: +- arrow +model-index: +- name: attempt4_20260310_173435 + results: [] +--- + + + +# attempt4_20260310_173435 + +This model is a fine-tuned version of [/data/jrlin/HyperScale/.cache/model_specs/qwen3_hyp/qwen3/base/std/p686m](https://huggingface.co//data/jrlin/HyperScale/.cache/model_specs/qwen3_hyp/qwen3/base/std/p686m) on the owt_local_train dataset. +It achieves the following results on the evaluation set: +- Loss: 3.0189 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 4 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 128 +- total_eval_batch_size: 32 +- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.02 +- training_steps: 32021 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | +|:-------------:|:------:|:-----:|:---------------:| +| 4.0785 | 0.0616 | 2048 | 4.0436 | +| 3.6423 | 0.1232 | 4096 | 3.6262 | +| 3.45 | 0.1848 | 6144 | 3.4438 | +| 3.3322 | 0.2464 | 8192 | 3.3362 | +| 3.2598 | 0.3081 | 10240 | 3.2609 | +| 3.1952 | 0.3697 | 12288 | 3.2035 | +| 3.1597 | 0.4313 | 14336 | 3.1584 | +| 3.1175 | 0.4929 | 16384 | 3.1230 | +| 3.0847 | 0.5545 | 18432 | 3.0984 | +| 3.0592 | 0.6161 | 20480 | 3.0699 | +| 3.045 | 0.6777 | 22528 | 3.0511 | +| 3.0309 | 0.7393 | 24576 | 3.0370 | +| 3.02 | 0.8010 | 26624 | 3.0271 | +| 3.0174 | 0.8626 | 28672 | 3.0214 | +| 3.0123 | 0.9242 | 30720 | 3.0191 | + + +### Framework versions + +- Transformers 4.51.3 +- Pytorch 2.5.1+cu124 +- Datasets 3.6.0 +- Tokenizers 0.21.4 diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/__init__.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..33350c83a2c161ee228677e8f6fc4b495e9c05bb --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/__init__.py @@ -0,0 +1,49 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from ..utils import _LazyModule + + +_import_structure = { + "config": [ + "EXTERNAL_DATA_FORMAT_SIZE_LIMIT", + "OnnxConfig", + "OnnxConfigWithPast", + "OnnxSeq2SeqConfigWithPast", + "PatchingSpec", + ], + "convert": ["export", "validate_model_outputs"], + "features": ["FeaturesManager"], + "utils": ["ParameterFormat", "compute_serialized_parameters_size"], +} + + +if TYPE_CHECKING: + from .config import ( + EXTERNAL_DATA_FORMAT_SIZE_LIMIT, + OnnxConfig, + OnnxConfigWithPast, + OnnxSeq2SeqConfigWithPast, + PatchingSpec, + ) + from .convert import export, validate_model_outputs + from .features import FeaturesManager + from .utils import ParameterFormat, compute_serialized_parameters_size + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/accelerate.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/accelerate.py new file mode 100644 index 0000000000000000000000000000000000000000..83efac9661af0423901f26783c76194aa7c205d6 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/accelerate.py @@ -0,0 +1,196 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Since, https://github.com/huggingface/transformers/pull/36963, loading is always performed with models on meta +device. But since the `init_empty_weights` and `find_tied_parameters` functions are from accelerate, and accelerate is +somewhat still a soft dependency, we copy the functions here to be used natively in Transformers. + +The `init_empty_weights` and `init_on_device` functions were copied from `accelerate.big_modeling.py`, and the +`find_tied_parameters` was copied from `accelerate.utils.modeling.py` +""" + +from contextlib import contextmanager + +from ..utils import is_torch_available, logging + + +if is_torch_available(): + import torch + import torch.nn as nn + + +logger = logging.get_logger(__name__) + + +@contextmanager +def init_empty_weights(include_buffers: bool = False): + """ + A context manager under which models are initialized with all parameters on the meta device, therefore creating an + empty model. Useful when just initializing the model would blow the available RAM. + + Args: + include_buffers (`bool`, *optional*): + Whether or not to also put all buffers on the meta device while initializing. + + Example: + + ```python + import torch.nn as nn + from accelerate import init_empty_weights + + # Initialize a model with 100 billions parameters in no time and without using any RAM. + with init_empty_weights(): + tst = nn.Sequential(*[nn.Linear(10000, 10000) for _ in range(1000)]) + ``` + + + + Any model created under this context manager has no weights. As such you can't do something like + `model.to(some_device)` with it. To load weights inside your empty model, see [`load_checkpoint_and_dispatch`]. + Make sure to overwrite the default device_map param for [`load_checkpoint_and_dispatch`], otherwise dispatch is not + called. + + + """ + with init_on_device(torch.device("meta"), include_buffers=include_buffers) as f: + yield f + + +@contextmanager +def init_on_device(device: "torch.device", include_buffers: bool = False): + """ + A context manager under which models are initialized with all parameters on the specified device. + + Args: + device (`torch.device`): + Device to initialize all parameters on. + include_buffers (`bool`, *optional*): + Whether or not to also put all buffers on the meta device while initializing. + + Example: + + ```python + import torch.nn as nn + from accelerate import init_on_device + + with init_on_device(device=torch.device("cuda")): + tst = nn.Linear(100, 100) # on `cuda` device + ``` + """ + if include_buffers: + with device: + yield + return + + old_register_parameter = nn.Module.register_parameter + if include_buffers: + old_register_buffer = nn.Module.register_buffer + + def register_empty_parameter(module, name, param): + old_register_parameter(module, name, param) + if param is not None: + param_cls = type(module._parameters[name]) + kwargs = module._parameters[name].__dict__ + kwargs["requires_grad"] = param.requires_grad + module._parameters[name] = param_cls(module._parameters[name].to(device), **kwargs) + + def register_empty_buffer(module, name, buffer, persistent=True): + old_register_buffer(module, name, buffer, persistent=persistent) + if buffer is not None: + module._buffers[name] = module._buffers[name].to(device) + + # Patch tensor creation + if include_buffers: + tensor_constructors_to_patch = { + torch_function_name: getattr(torch, torch_function_name) + for torch_function_name in ["empty", "zeros", "ones", "full"] + } + else: + tensor_constructors_to_patch = {} + + def patch_tensor_constructor(fn): + def wrapper(*args, **kwargs): + kwargs["device"] = device + return fn(*args, **kwargs) + + return wrapper + + try: + nn.Module.register_parameter = register_empty_parameter + if include_buffers: + nn.Module.register_buffer = register_empty_buffer + for torch_function_name in tensor_constructors_to_patch.keys(): + setattr(torch, torch_function_name, patch_tensor_constructor(getattr(torch, torch_function_name))) + yield + finally: + nn.Module.register_parameter = old_register_parameter + if include_buffers: + nn.Module.register_buffer = old_register_buffer + for torch_function_name, old_torch_function in tensor_constructors_to_patch.items(): + setattr(torch, torch_function_name, old_torch_function) + + +def find_tied_parameters(model: "nn.Module", **kwargs): + """ + Find the tied parameters in a given model. + + + + The signature accepts keyword arguments, but they are for the recursive part of this function and you should ignore + them. + + + + Args: + model (`torch.nn.Module`): The model to inspect. + + Returns: + List[List[str]]: A list of lists of parameter names being all tied together. + + Example: + + ```py + >>> from collections import OrderedDict + >>> import torch.nn as nn + + >>> model = nn.Sequential(OrderedDict([("linear1", nn.Linear(4, 4)), ("linear2", nn.Linear(4, 4))])) + >>> model.linear2.weight = model.linear1.weight + >>> find_tied_parameters(model) + [['linear1.weight', 'linear2.weight']] + ``` + """ + + # get ALL model parameters and thier names + all_named_parameters = dict(model.named_parameters(remove_duplicate=False)) + + # get ONLY unique named parameters, + # if parameter is tied and have multiple names, it will be included only once + no_duplicate_named_parameters = dict(model.named_parameters(remove_duplicate=True)) + + # the difference of the two sets will give us the tied parameters + tied_param_names = set(all_named_parameters.keys()) - set(no_duplicate_named_parameters.keys()) + + # 'tied_param_names' contains the names of parameters that are tied in the model, but we do not know + # which names refer to the same parameter. To identify this, we need to group them together. + tied_param_groups = {} + for tied_param_name in tied_param_names: + tied_param = all_named_parameters[tied_param_name] + for param_name, param in no_duplicate_named_parameters.items(): + # compare if parameters are the same, if so, group thier names together + if param is tied_param: + if param_name not in tied_param_groups: + tied_param_groups[param_name] = [] + tied_param_groups[param_name].append(tied_param_name) + + return [sorted([weight] + list(set(tied))) for weight, tied in tied_param_groups.items()] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/activations.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/activations.py new file mode 100644 index 0000000000000000000000000000000000000000..15f0397535e8c6df24cbd2148471ed71742d0636 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/activations.py @@ -0,0 +1,240 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from collections import OrderedDict + +import torch +from packaging import version +from torch import Tensor, nn + +from .utils import logging + + +logger = logging.get_logger(__name__) + + +class PytorchGELUTanh(nn.Module): + """ + A fast C implementation of the tanh approximation of the GeLU activation function. See + https://arxiv.org/abs/1606.08415. + + This implementation is equivalent to NewGELU and FastGELU but much faster. However, it is not an exact numerical + match due to rounding errors. + """ + + def __init__(self): + super().__init__() + if version.parse(torch.__version__) < version.parse("1.12.0"): + raise ImportError( + f"You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use " + "PytorchGELUTanh. Please upgrade torch." + ) + + def forward(self, input: Tensor) -> Tensor: + return nn.functional.gelu(input, approximate="tanh") + + +class NewGELUActivation(nn.Module): + """ + Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see + the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415 + """ + + def forward(self, input: Tensor) -> Tensor: + return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.044715 * torch.pow(input, 3.0)))) + + +class GELUActivation(nn.Module): + """ + Original Implementation of the GELU activation function in Google BERT repo when initially created. For + information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 + + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional + Also see the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415 + """ + + def __init__(self, use_gelu_python: bool = False): + super().__init__() + if use_gelu_python: + self.act = self._gelu_python + else: + self.act = nn.functional.gelu + + def _gelu_python(self, input: Tensor) -> Tensor: + return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0))) + + def forward(self, input: Tensor) -> Tensor: + return self.act(input) + + +class FastGELUActivation(nn.Module): + """ + Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs + """ + + def forward(self, input: Tensor) -> Tensor: + return 0.5 * input * (1.0 + torch.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input))) + + +class QuickGELUActivation(nn.Module): + """ + Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs + """ + + def forward(self, input: Tensor) -> Tensor: + return input * torch.sigmoid(1.702 * input) + + +class ClippedGELUActivation(nn.Module): + """ + Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as + it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to + https://arxiv.org/abs/2004.09602. + + Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when + initially created. + + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://arxiv.org/abs/1606.08415 + """ + + def __init__(self, min: float, max: float): + if min > max: + raise ValueError(f"min should be < max (got min: {min}, max: {max})") + + super().__init__() + self.min = min + self.max = max + + def forward(self, x: Tensor) -> Tensor: + return torch.clip(gelu(x), self.min, self.max) + + +class AccurateGELUActivation(nn.Module): + """ + Applies GELU approximation that is faster than default and more accurate than QuickGELU. See: + https://github.com/hendrycks/GELUs + + Implemented along with MEGA (Moving Average Equipped Gated Attention) + """ + + def __init__(self): + super().__init__() + self.precomputed_constant = math.sqrt(2 / math.pi) + + def forward(self, input: Tensor) -> Tensor: + return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044715 * torch.pow(input, 3)))) + + +class MishActivation(nn.Module): + """ + See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://arxiv.org/abs/1908.08681). Also + visit the official repository for the paper: https://github.com/digantamisra98/Mish + """ + + def __init__(self): + super().__init__() + if version.parse(torch.__version__) < version.parse("1.9.0"): + self.act = self._mish_python + else: + self.act = nn.functional.mish + + def _mish_python(self, input: Tensor) -> Tensor: + return input * torch.tanh(nn.functional.softplus(input)) + + def forward(self, input: Tensor) -> Tensor: + return self.act(input) + + +class LinearActivation(nn.Module): + """ + Applies the linear activation function, i.e. forwarding input directly to output. + """ + + def forward(self, input: Tensor) -> Tensor: + return input + + +class LaplaceActivation(nn.Module): + """ + Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See + https://arxiv.org/abs/2209.10655 + + Inspired by squared relu, but with bounded range and gradient for better stability + """ + + def forward(self, input, mu=0.707107, sigma=0.282095): + input = (input - mu).div(sigma * math.sqrt(2.0)) + return 0.5 * (1.0 + torch.erf(input)) + + +class ReLUSquaredActivation(nn.Module): + """ + Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2 + """ + + def forward(self, input): + relu_applied = nn.functional.relu(input) + squared = torch.square(relu_applied) + return squared + + +class ClassInstantier(OrderedDict): + def __getitem__(self, key): + content = super().__getitem__(key) + cls, kwargs = content if isinstance(content, tuple) else (content, {}) + return cls(**kwargs) + + +ACT2CLS = { + "gelu": GELUActivation, + "gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}), + "gelu_fast": FastGELUActivation, + "gelu_new": NewGELUActivation, + "gelu_python": (GELUActivation, {"use_gelu_python": True}), + "gelu_pytorch_tanh": PytorchGELUTanh, + "gelu_accurate": AccurateGELUActivation, + "laplace": LaplaceActivation, + "leaky_relu": nn.LeakyReLU, + "linear": LinearActivation, + "mish": MishActivation, + "quick_gelu": QuickGELUActivation, + "relu": nn.ReLU, + "relu2": ReLUSquaredActivation, + "relu6": nn.ReLU6, + "sigmoid": nn.Sigmoid, + "silu": nn.SiLU, + "swish": nn.SiLU, + "tanh": nn.Tanh, + "prelu": nn.PReLU, +} +ACT2FN = ClassInstantier(ACT2CLS) + + +def get_activation(activation_string): + if activation_string in ACT2FN: + return ACT2FN[activation_string] + else: + raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}") + + +# For backwards compatibility with: from activations import gelu_python +gelu_python = get_activation("gelu_python") +gelu_new = get_activation("gelu_new") +gelu = get_activation("gelu") +gelu_fast = get_activation("gelu_fast") +quick_gelu = get_activation("quick_gelu") +silu = get_activation("silu") +mish = get_activation("mish") +linear_act = get_activation("linear") diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/activations_tf.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/activations_tf.py new file mode 100644 index 0000000000000000000000000000000000000000..d12b73ea45176f3a4bc42cdabe8b73078a3b90f2 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/activations_tf.py @@ -0,0 +1,147 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import tensorflow as tf +from packaging.version import parse + + +try: + import tf_keras as keras +except (ModuleNotFoundError, ImportError): + import keras + + if parse(keras.__version__).major > 2: + raise ValueError( + "Your currently installed version of Keras is Keras 3, but this is not yet supported in " + "Transformers. Please install the backwards-compatible tf-keras package with " + "`pip install tf-keras`." + ) + + +def _gelu(x): + """ + Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when + initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see + https://arxiv.org/abs/1606.08415 + """ + x = tf.convert_to_tensor(x) + cdf = 0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0), x.dtype))) + + return x * cdf + + +def _gelu_new(x): + """ + Gaussian Error Linear Unit. This is a smoother version of the GELU. Original paper: https://arxiv.org/abs/1606.0841 + + Args: + x: float Tensor to perform activation + + Returns: + `x` with the GELU activation applied. + """ + x = tf.convert_to_tensor(x) + pi = tf.cast(math.pi, x.dtype) + coeff = tf.cast(0.044715, x.dtype) + cdf = 0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi) * (x + coeff * tf.pow(x, 3)))) + + return x * cdf + + +def mish(x): + x = tf.convert_to_tensor(x) + + return x * tf.tanh(tf.math.softplus(x)) + + +def gelu_fast(x): + x = tf.convert_to_tensor(x) + coeff1 = tf.cast(0.044715, x.dtype) + coeff2 = tf.cast(0.7978845608, x.dtype) + + return 0.5 * x * (1.0 + tf.tanh(x * coeff2 * (1.0 + coeff1 * x * x))) + + +def quick_gelu(x): + x = tf.convert_to_tensor(x) + coeff = tf.cast(1.702, x.dtype) + return x * tf.math.sigmoid(coeff * x) + + +def gelu_10(x): + """ + Clip the range of possible GeLU outputs between [-10, 10]. This is especially useful for quantization purpose, as + it allows mapping 2 negatives values in the GeLU spectrum. For more information on this trick, please refer to + https://arxiv.org/abs/2004.09602 + + Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when + initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see + https://arxiv.org/abs/1606.08415 :param x: :return: + """ + return tf.clip_by_value(_gelu(x), -10, 10) + + +def glu(x, axis=-1): + """ + Gated Linear Unit. Implementation as defined in the original paper (see https://arxiv.org/abs/1612.08083), where + the input `x` is split in two halves across a dimension (`axis`), A and B, returning A * sigmoid(B). + + Args: + `x`: float Tensor to perform activation + `axis`: dimension across which `x` be split in half + + Returns: + `x` with the GLU activation applied (with its size halved across the dimension `axis`). + """ + a, b = tf.split(x, 2, axis=axis) + return a * tf.math.sigmoid(b) + + +if parse(tf.version.VERSION) >= parse("2.4"): + + def approximate_gelu_wrap(x): + return keras.activations.gelu(x, approximate=True) + + gelu = keras.activations.gelu + gelu_new = approximate_gelu_wrap +else: + gelu = _gelu + gelu_new = _gelu_new + + +ACT2FN = { + "gelu": gelu, + "gelu_10": gelu_10, + "gelu_fast": gelu_fast, + "gelu_new": gelu_new, + "glu": glu, + "mish": mish, + "quick_gelu": quick_gelu, + "relu": keras.activations.relu, + "sigmoid": keras.activations.sigmoid, + "silu": keras.activations.swish, + "swish": keras.activations.swish, + "tanh": keras.activations.tanh, +} + + +def get_tf_activation(activation_string): + if activation_string in ACT2FN: + return ACT2FN[activation_string] + else: + raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}") diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/added_tokens.json b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/all_results.json b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..4138e48287801695a88472a218c514e6b8f28067 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/all_results.json @@ -0,0 +1,13 @@ +{ + "epoch": 0.9633128262210255, + "eval_loss": 3.0188777446746826, + "eval_perplexity": 20.46830812210243, + "eval_runtime": 18.5237, + "eval_samples_per_second": 228.788, + "eval_steps_per_second": 7.18, + "total_flos": 2.4758148444989686e+19, + "train_loss": 3.3313790206358793, + "train_runtime": 74724.9408, + "train_samples_per_second": 54.85, + "train_steps_per_second": 0.429 +} \ No newline at end of file diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/aqlm.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/aqlm.py new file mode 100644 index 0000000000000000000000000000000000000000..0626da7aced5bc259295ac51c58931858aac8dc8 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/aqlm.py @@ -0,0 +1,100 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"AQLM (Additive Quantization of Language Model) integration file" + +from ..utils import ACCELERATE_MIN_VERSION, is_accelerate_available, is_aqlm_available, is_torch_available + + +if is_torch_available(): + import torch.nn as nn + + +def replace_with_aqlm_linear( + model, + quantization_config=None, + linear_weights_not_to_quantize=None, + current_key_name=None, + has_been_replaced=False, +): + """ + Public method that recursively replaces the Linear layers of the given model with AQLM quantized layers. + `accelerate` is needed to use this method. Returns the converted model and a boolean that indicates if the + conversion has been successfull or not. + + Args: + model (`torch.nn.Module`): + The model to convert, can be any `torch.nn.Module` instance. + quantization_config (`AqlmConfig`): + The quantization config object that contains the quantization parameters. + linear_weights_not_to_quantize (`list[str]`, *optional*): + A list of nn.Linear weights to not convert. If a parameter path is in the list (e.g. `lm_head.weight`), the corresponding module will not be + converted. + current_key_name (`list`, *optional*): + A list that contains the current key name. This is used for recursion and should not be passed by the user. + has_been_replaced (`bool`, *optional*): + A boolean that indicates if the conversion has been successful or not. This is used for recursion and + should not be passed by the user. + """ + if not is_aqlm_available(): + raise ValueError("AQLM is not available. Please install it with `pip install aqlm[cpu,gpu]`") + + if not is_accelerate_available(): + raise ValueError( + f"AQLM requires Accelerate to be installed: `pip install 'accelerate>={ACCELERATE_MIN_VERSION}'`" + ) + + if linear_weights_not_to_quantize is None: + linear_weights_not_to_quantize = [] + + from accelerate import init_empty_weights + from aqlm import QuantizedLinear + + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + if isinstance(module, nn.Linear): + # Check if the current key is not in the `linear_weights_not_to_quantize` + if ".".join(current_key_name) + ".weight" not in linear_weights_not_to_quantize: + with init_empty_weights(): + in_features = module.in_features + out_features = module.out_features + + model._modules[name] = QuantizedLinear( + in_features, + out_features, + bias=module.bias is not None, + in_group_size=quantization_config.in_group_size, + out_group_size=quantization_config.out_group_size, + num_codebooks=quantization_config.num_codebooks, + nbits_per_codebook=quantization_config.nbits_per_codebook, + ) + has_been_replaced = True + + # Store the module class in case we need to transpose the weight later + model._modules[name].source_cls = type(module) + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) + if len(list(module.children())) > 0: + _, has_been_replaced = replace_with_aqlm_linear( + module, + quantization_config=quantization_config, + linear_weights_not_to_quantize=linear_weights_not_to_quantize, + current_key_name=current_key_name, + has_been_replaced=has_been_replaced, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/audio_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/audio_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8420a84e089e03be9a80fb63c237e34203ea28a0 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/audio_utils.py @@ -0,0 +1,1193 @@ +# Copyright 2023 The HuggingFace Inc. team and the librosa & torchaudio authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Audio processing functions to extract features from audio waveforms. This code is pure numpy to support all frameworks +and remove unnecessary dependencies. +""" + +import os +import warnings +from io import BytesIO +from typing import List, Optional, Tuple, Union + +import numpy as np +import requests + +from .utils import is_librosa_available, requires_backends + + +if is_librosa_available(): + import librosa + + +def load_audio(audio: Union[str, np.ndarray], sampling_rate=16000, timeout=None) -> np.ndarray: + """ + Loads `audio` to an np.ndarray object. + + Args: + audio (`str` or `np.ndarray`): + The audio to be laoded to the numpy array format. + sampling_rate (`int`, *optional*, defaults to 16000): + The samlping rate to be used when loading the audio. It should be same as the + sampling rate the model you will be using further was trained with. + timeout (`float`, *optional*): + The timeout value in seconds for the URL request. + + Returns: + `np.ndarray`: A numpy artay representing the audio. + """ + requires_backends(load_audio, ["librosa"]) + + if isinstance(audio, str): + # Load audio from URL (e.g https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/translate_to_chinese.wav) + if audio.startswith("http://") or audio.startswith("https://"): + audio = librosa.load(BytesIO(requests.get(audio, timeout=timeout).content), sr=sampling_rate)[0] + elif os.path.isfile(audio): + audio = librosa.load(audio, sr=sampling_rate)[0] + elif isinstance(audio, np.ndarray): + audio = audio + else: + raise TypeError( + "Incorrect format used for `audio`. Should be an url linking to an audio, a local path, or numpy array." + ) + return audio + + +AudioInput = Union[ + np.ndarray, "torch.Tensor", List[np.ndarray], Tuple[np.ndarray], List["torch.Tensor"], Tuple["torch.Tensor"] # noqa: F821 +] + + +def hertz_to_mel(freq: Union[float, np.ndarray], mel_scale: str = "htk") -> Union[float, np.ndarray]: + """ + Convert frequency from hertz to mels. + + Args: + freq (`float` or `np.ndarray`): + The frequency, or multiple frequencies, in hertz (Hz). + mel_scale (`str`, *optional*, defaults to `"htk"`): + The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`. + + Returns: + `float` or `np.ndarray`: The frequencies on the mel scale. + """ + + if mel_scale not in ["slaney", "htk", "kaldi"]: + raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".') + + if mel_scale == "htk": + return 2595.0 * np.log10(1.0 + (freq / 700.0)) + elif mel_scale == "kaldi": + return 1127.0 * np.log(1.0 + (freq / 700.0)) + + min_log_hertz = 1000.0 + min_log_mel = 15.0 + logstep = 27.0 / np.log(6.4) + mels = 3.0 * freq / 200.0 + + if isinstance(freq, np.ndarray): + log_region = freq >= min_log_hertz + mels[log_region] = min_log_mel + np.log(freq[log_region] / min_log_hertz) * logstep + elif freq >= min_log_hertz: + mels = min_log_mel + np.log(freq / min_log_hertz) * logstep + + return mels + + +def mel_to_hertz(mels: Union[float, np.ndarray], mel_scale: str = "htk") -> Union[float, np.ndarray]: + """ + Convert frequency from mels to hertz. + + Args: + mels (`float` or `np.ndarray`): + The frequency, or multiple frequencies, in mels. + mel_scale (`str`, *optional*, `"htk"`): + The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`. + + Returns: + `float` or `np.ndarray`: The frequencies in hertz. + """ + + if mel_scale not in ["slaney", "htk", "kaldi"]: + raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".') + + if mel_scale == "htk": + return 700.0 * (np.power(10, mels / 2595.0) - 1.0) + elif mel_scale == "kaldi": + return 700.0 * (np.exp(mels / 1127.0) - 1.0) + + min_log_hertz = 1000.0 + min_log_mel = 15.0 + logstep = np.log(6.4) / 27.0 + freq = 200.0 * mels / 3.0 + + if isinstance(mels, np.ndarray): + log_region = mels >= min_log_mel + freq[log_region] = min_log_hertz * np.exp(logstep * (mels[log_region] - min_log_mel)) + elif mels >= min_log_mel: + freq = min_log_hertz * np.exp(logstep * (mels - min_log_mel)) + + return freq + + +def hertz_to_octave( + freq: Union[float, np.ndarray], tuning: Optional[float] = 0.0, bins_per_octave: Optional[int] = 12 +): + """ + Convert frequency from hertz to fractional octave numbers. + Adapted from *librosa*. + + Args: + freq (`float` or `np.ndarray`): + The frequency, or multiple frequencies, in hertz (Hz). + tuning (`float`, defaults to `0.`): + Tuning deviation from the Stuttgart pitch (A440) in (fractional) bins per octave. + bins_per_octave (`int`, defaults to `12`): + Number of bins per octave. + + Returns: + `float` or `np.ndarray`: The frequencies on the octave scale. + """ + stuttgart_pitch = 440.0 * 2.0 ** (tuning / bins_per_octave) + octave = np.log2(freq / (float(stuttgart_pitch) / 16)) + return octave + + +def _create_triangular_filter_bank(fft_freqs: np.ndarray, filter_freqs: np.ndarray) -> np.ndarray: + """ + Creates a triangular filter bank. + + Adapted from *torchaudio* and *librosa*. + + Args: + fft_freqs (`np.ndarray` of shape `(num_frequency_bins,)`): + Discrete frequencies of the FFT bins in Hz. + filter_freqs (`np.ndarray` of shape `(num_mel_filters,)`): + Center frequencies of the triangular filters to create, in Hz. + + Returns: + `np.ndarray` of shape `(num_frequency_bins, num_mel_filters)` + """ + filter_diff = np.diff(filter_freqs) + slopes = np.expand_dims(filter_freqs, 0) - np.expand_dims(fft_freqs, 1) + down_slopes = -slopes[:, :-2] / filter_diff[:-1] + up_slopes = slopes[:, 2:] / filter_diff[1:] + return np.maximum(np.zeros(1), np.minimum(down_slopes, up_slopes)) + + +def chroma_filter_bank( + num_frequency_bins: int, + num_chroma: int, + sampling_rate: int, + tuning: float = 0.0, + power: Optional[float] = 2.0, + weighting_parameters: Optional[tuple[float, float]] = (5.0, 2.0), + start_at_c_chroma: Optional[bool] = True, +): + """ + Creates a chroma filter bank, i.e a linear transformation to project spectrogram bins onto chroma bins. + + Adapted from *librosa*. + + Args: + num_frequency_bins (`int`): + Number of frequencies used to compute the spectrogram (should be the same as in `stft`). + num_chroma (`int`): + Number of chroma bins (i.e pitch classes). + sampling_rate (`float`): + Sample rate of the audio waveform. + tuning (`float`): + Tuning deviation from A440 in fractions of a chroma bin. + power (`float`, *optional*, defaults to 2.0): + If 12.0, normalizes each column with their L2 norm. If 1.0, normalizes each column with their L1 norm. + weighting_parameters (`Tuple[float, float]`, *optional*, defaults to `(5., 2.)`): + If specified, apply a Gaussian weighting parameterized by the first element of the tuple being the center and + the second element being the Gaussian half-width. + start_at_c_chroma (`float`, *optional*, defaults to `True`): + If True, the filter bank will start at the 'C' pitch class. Otherwise, it will start at 'A'. + Returns: + `np.ndarray` of shape `(num_frequency_bins, num_chroma)` + """ + # Get the FFT bins, not counting the DC component + frequencies = np.linspace(0, sampling_rate, num_frequency_bins, endpoint=False)[1:] + + freq_bins = num_chroma * hertz_to_octave(frequencies, tuning=tuning, bins_per_octave=num_chroma) + + # make up a value for the 0 Hz bin = 1.5 octaves below bin 1 + # (so chroma is 50% rotated from bin 1, and bin width is broad) + freq_bins = np.concatenate(([freq_bins[0] - 1.5 * num_chroma], freq_bins)) + + bins_width = np.concatenate((np.maximum(freq_bins[1:] - freq_bins[:-1], 1.0), [1])) + + chroma_filters = np.subtract.outer(freq_bins, np.arange(0, num_chroma, dtype="d")).T + + num_chroma2 = np.round(float(num_chroma) / 2) + + # Project into range -num_chroma/2 .. num_chroma/2 + # add on fixed offset of 10*num_chroma to ensure all values passed to + # rem are positive + chroma_filters = np.remainder(chroma_filters + num_chroma2 + 10 * num_chroma, num_chroma) - num_chroma2 + + # Gaussian bumps - 2*D to make them narrower + chroma_filters = np.exp(-0.5 * (2 * chroma_filters / np.tile(bins_width, (num_chroma, 1))) ** 2) + + # normalize each column + if power is not None: + chroma_filters = chroma_filters / np.sum(chroma_filters**power, axis=0, keepdims=True) ** (1.0 / power) + + # Maybe apply scaling for fft bins + if weighting_parameters is not None: + center, half_width = weighting_parameters + chroma_filters *= np.tile( + np.exp(-0.5 * (((freq_bins / num_chroma - center) / half_width) ** 2)), + (num_chroma, 1), + ) + + if start_at_c_chroma: + chroma_filters = np.roll(chroma_filters, -3 * (num_chroma // 12), axis=0) + + # remove aliasing columns, copy to ensure row-contiguity + return np.ascontiguousarray(chroma_filters[:, : int(1 + num_frequency_bins / 2)]) + + +def mel_filter_bank( + num_frequency_bins: int, + num_mel_filters: int, + min_frequency: float, + max_frequency: float, + sampling_rate: int, + norm: Optional[str] = None, + mel_scale: str = "htk", + triangularize_in_mel_space: bool = False, +) -> np.ndarray: + """ + Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and + various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters + are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these + features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency. + + Different banks of mel filters were introduced in the literature. The following variations are supported: + + - MFCC FB-20: introduced in 1980 by Davis and Mermelstein, it assumes a sampling frequency of 10 kHz and a speech + bandwidth of `[0, 4600]` Hz. + - MFCC FB-24 HTK: from the Cambridge HMM Toolkit (HTK) (1995) uses a filter bank of 24 filters for a speech + bandwidth of `[0, 8000]` Hz. This assumes sampling rate ≥ 16 kHz. + - MFCC FB-40: from the Auditory Toolbox for MATLAB written by Slaney in 1998, assumes a sampling rate of 16 kHz and + speech bandwidth of `[133, 6854]` Hz. This version also includes area normalization. + - HFCC-E FB-29 (Human Factor Cepstral Coefficients) of Skowronski and Harris (2004), assumes a sampling rate of + 12.5 kHz and speech bandwidth of `[0, 6250]` Hz. + + This code is adapted from *torchaudio* and *librosa*. Note that the default parameters of torchaudio's + `melscale_fbanks` implement the `"htk"` filters while librosa uses the `"slaney"` implementation. + + Args: + num_frequency_bins (`int`): + Number of frequency bins (should be the same as `n_fft // 2 + 1` where `n_fft` is the size of the Fourier Transform used to compute the spectrogram). + num_mel_filters (`int`): + Number of mel filters to generate. + min_frequency (`float`): + Lowest frequency of interest in Hz. + max_frequency (`float`): + Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`. + sampling_rate (`int`): + Sample rate of the audio waveform. + norm (`str`, *optional*): + If `"slaney"`, divide the triangular mel weights by the width of the mel band (area normalization). + mel_scale (`str`, *optional*, defaults to `"htk"`): + The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`. + triangularize_in_mel_space (`bool`, *optional*, defaults to `False`): + If this option is enabled, the triangular filter is applied in mel space rather than frequency space. This + should be set to `true` in order to get the same results as `torchaudio` when computing mel filters. + + Returns: + `np.ndarray` of shape (`num_frequency_bins`, `num_mel_filters`): Triangular filter bank matrix. This is a + projection matrix to go from a spectrogram to a mel spectrogram. + """ + if norm is not None and norm != "slaney": + raise ValueError('norm must be one of None or "slaney"') + + if num_frequency_bins < 2: + raise ValueError(f"Require num_frequency_bins: {num_frequency_bins} >= 2") + + if min_frequency > max_frequency: + raise ValueError(f"Require min_frequency: {min_frequency} <= max_frequency: {max_frequency}") + + # center points of the triangular mel filters + mel_min = hertz_to_mel(min_frequency, mel_scale=mel_scale) + mel_max = hertz_to_mel(max_frequency, mel_scale=mel_scale) + mel_freqs = np.linspace(mel_min, mel_max, num_mel_filters + 2) + filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_scale) + + if triangularize_in_mel_space: + # frequencies of FFT bins in Hz, but filters triangularized in mel space + fft_bin_width = sampling_rate / ((num_frequency_bins - 1) * 2) + fft_freqs = hertz_to_mel(fft_bin_width * np.arange(num_frequency_bins), mel_scale=mel_scale) + filter_freqs = mel_freqs + else: + # frequencies of FFT bins in Hz + fft_freqs = np.linspace(0, sampling_rate // 2, num_frequency_bins) + + mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs) + + if norm is not None and norm == "slaney": + # Slaney-style mel is scaled to be approx constant energy per channel + enorm = 2.0 / (filter_freqs[2 : num_mel_filters + 2] - filter_freqs[:num_mel_filters]) + mel_filters *= np.expand_dims(enorm, 0) + + if (mel_filters.max(axis=0) == 0.0).any(): + warnings.warn( + "At least one mel filter has all zero values. " + f"The value for `num_mel_filters` ({num_mel_filters}) may be set too high. " + f"Or, the value for `num_frequency_bins` ({num_frequency_bins}) may be set too low." + ) + + return mel_filters + + +def optimal_fft_length(window_length: int) -> int: + """ + Finds the best FFT input size for a given `window_length`. This function takes a given window length and, if not + already a power of two, rounds it up to the next power or two. + + The FFT algorithm works fastest when the length of the input is a power of two, which may be larger than the size + of the window or analysis frame. For example, if the window is 400 samples, using an FFT input size of 512 samples + is more optimal than an FFT size of 400 samples. Using a larger FFT size does not affect the detected frequencies, + it simply gives a higher frequency resolution (i.e. the frequency bins are smaller). + """ + return 2 ** int(np.ceil(np.log2(window_length))) + + +def window_function( + window_length: int, + name: str = "hann", + periodic: bool = True, + frame_length: Optional[int] = None, + center: bool = True, +) -> np.ndarray: + """ + Returns an array containing the specified window. This window is intended to be used with `stft`. + + The following window types are supported: + + - `"boxcar"`: a rectangular window + - `"hamming"`: the Hamming window + - `"hann"`: the Hann window + - `"povey"`: the Povey window + + Args: + window_length (`int`): + The length of the window in samples. + name (`str`, *optional*, defaults to `"hann"`): + The name of the window function. + periodic (`bool`, *optional*, defaults to `True`): + Whether the window is periodic or symmetric. + frame_length (`int`, *optional*): + The length of the analysis frames in samples. Provide a value for `frame_length` if the window is smaller + than the frame length, so that it will be zero-padded. + center (`bool`, *optional*, defaults to `True`): + Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided. + + Returns: + `np.ndarray` of shape `(window_length,)` or `(frame_length,)` containing the window. + """ + length = window_length + 1 if periodic else window_length + + if name == "boxcar": + window = np.ones(length) + elif name in ["hamming", "hamming_window"]: + window = np.hamming(length) + elif name in ["hann", "hann_window"]: + window = np.hanning(length) + elif name in ["povey"]: + window = np.power(np.hanning(length), 0.85) + else: + raise ValueError(f"Unknown window function '{name}'") + + if periodic: + window = window[:-1] + + if frame_length is None: + return window + + if window_length > frame_length: + raise ValueError( + f"Length of the window ({window_length}) may not be larger than frame_length ({frame_length})" + ) + + padded_window = np.zeros(frame_length) + offset = (frame_length - window_length) // 2 if center else 0 + padded_window[offset : offset + window_length] = window + return padded_window + + +# TODO This method does not support batching yet as we are mainly focused on inference. +def spectrogram( + waveform: np.ndarray, + window: np.ndarray, + frame_length: int, + hop_length: int, + fft_length: Optional[int] = None, + power: Optional[float] = 1.0, + center: bool = True, + pad_mode: str = "reflect", + onesided: bool = True, + dither: float = 0.0, + preemphasis: Optional[float] = None, + mel_filters: Optional[np.ndarray] = None, + mel_floor: float = 1e-10, + log_mel: Optional[str] = None, + reference: float = 1.0, + min_value: float = 1e-10, + db_range: Optional[float] = None, + remove_dc_offset: Optional[bool] = None, + dtype: np.dtype = np.float32, +) -> np.ndarray: + """ + Calculates a spectrogram over one waveform using the Short-Time Fourier Transform. + + This function can create the following kinds of spectrograms: + + - amplitude spectrogram (`power = 1.0`) + - power spectrogram (`power = 2.0`) + - complex-valued spectrogram (`power = None`) + - log spectrogram (use `log_mel` argument) + - mel spectrogram (provide `mel_filters`) + - log-mel spectrogram (provide `mel_filters` and `log_mel`) + + How this works: + + 1. The input waveform is split into frames of size `frame_length` that are partially overlapping by `frame_length + - hop_length` samples. + 2. Each frame is multiplied by the window and placed into a buffer of size `fft_length`. + 3. The DFT is taken of each windowed frame. + 4. The results are stacked into a spectrogram. + + We make a distinction between the following "blocks" of sample data, each of which may have a different lengths: + + - The analysis frame. This is the size of the time slices that the input waveform is split into. + - The window. Each analysis frame is multiplied by the window to avoid spectral leakage. + - The FFT input buffer. The length of this determines how many frequency bins are in the spectrogram. + + In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. A + padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, + typically the next power of two. + + Note: This function is not optimized for speed yet. It should be mostly compatible with `librosa.stft` and + `torchaudio.functional.transforms.Spectrogram`, although it is more flexible due to the different ways spectrograms + can be constructed. + + Args: + waveform (`np.ndarray` of shape `(length,)`): + The input waveform. This must be a single real-valued, mono waveform. + window (`np.ndarray` of shape `(frame_length,)`): + The windowing function to apply, including zero-padding if necessary. The actual window length may be + shorter than `frame_length`, but we're assuming the array has already been zero-padded. + frame_length (`int`): + The length of the analysis frames in samples. With librosa this is always equal to `fft_length` but we also + allow smaller sizes. + hop_length (`int`): + The stride between successive analysis frames in samples. + fft_length (`int`, *optional*): + The size of the FFT buffer in samples. This determines how many frequency bins the spectrogram will have. + For optimal speed, this should be a power of two. If `None`, uses `frame_length`. + power (`float`, *optional*, defaults to 1.0): + If 1.0, returns the amplitude spectrogram. If 2.0, returns the power spectrogram. If `None`, returns + complex numbers. + center (`bool`, *optional*, defaults to `True`): + Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `False`, frame + `t` will start at time `t * hop_length`. + pad_mode (`str`, *optional*, defaults to `"reflect"`): + Padding mode used when `center` is `True`. Possible values are: `"constant"` (pad with zeros), `"edge"` + (pad with edge values), `"reflect"` (pads with mirrored values). + onesided (`bool`, *optional*, defaults to `True`): + If True, only computes the positive frequencies and returns a spectrogram containing `fft_length // 2 + 1` + frequency bins. If False, also computes the negative frequencies and returns `fft_length` frequency bins. + dither (`float`, *optional*, defaults to 0.0): + Adds dithering. In other words, adds a small Gaussian noise to each frame. + E.g. use 4.0 to add dithering with a normal distribution centered + around 0.0 with standard deviation 4.0, 0.0 means no dithering. + Dithering has similar effect as `mel_floor`. It reduces the high log_mel_fbank + values for signals with hard-zero sections, when VAD cutoff is present in the signal. + preemphasis (`float`, *optional*) + Coefficient for a low-pass filter that applies pre-emphasis before the DFT. + mel_filters (`np.ndarray` of shape `(num_freq_bins, num_mel_filters)`, *optional*): + The mel filter bank. If supplied, applies a this filter bank to create a mel spectrogram. + mel_floor (`float`, *optional*, defaults to 1e-10): + Minimum value of mel frequency banks. + log_mel (`str`, *optional*): + How to convert the spectrogram to log scale. Possible options are: `None` (don't convert), `"log"` (take + the natural logarithm) `"log10"` (take the base-10 logarithm), `"dB"` (convert to decibels). Can only be + used when `power` is not `None`. + reference (`float`, *optional*, defaults to 1.0): + Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set + the loudest part to 0 dB. Must be greater than zero. + min_value (`float`, *optional*, defaults to `1e-10`): + The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking + `log(0)`. For a power spectrogram, the default of `1e-10` corresponds to a minimum of -100 dB. For an + amplitude spectrogram, the value `1e-5` corresponds to -100 dB. Must be greater than zero. + db_range (`float`, *optional*): + Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the + peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + remove_dc_offset (`bool`, *optional*): + Subtract mean from waveform on each frame, applied before pre-emphasis. This should be set to `true` in + order to get the same results as `torchaudio.compliance.kaldi.fbank` when computing mel filters. + dtype (`np.dtype`, *optional*, defaults to `np.float32`): + Data type of the spectrogram tensor. If `power` is None, this argument is ignored and the dtype will be + `np.complex64`. + + Returns: + `nd.array` containing a spectrogram of shape `(num_frequency_bins, length)` for a regular spectrogram or shape + `(num_mel_filters, length)` for a mel spectrogram. + """ + window_length = len(window) + + if fft_length is None: + fft_length = frame_length + + if frame_length > fft_length: + raise ValueError(f"frame_length ({frame_length}) may not be larger than fft_length ({fft_length})") + + if window_length != frame_length: + raise ValueError(f"Length of the window ({window_length}) must equal frame_length ({frame_length})") + + if hop_length <= 0: + raise ValueError("hop_length must be greater than zero") + + if waveform.ndim != 1: + raise ValueError(f"Input waveform must have only one dimension, shape is {waveform.shape}") + + if np.iscomplexobj(waveform): + raise ValueError("Complex-valued input waveforms are not currently supported") + + if power is None and mel_filters is not None: + raise ValueError( + "You have provided `mel_filters` but `power` is `None`. Mel spectrogram computation is not yet supported for complex-valued spectrogram." + "Specify `power` to fix this issue." + ) + + # center pad the waveform + if center: + padding = [(int(frame_length // 2), int(frame_length // 2))] + waveform = np.pad(waveform, padding, mode=pad_mode) + + # promote to float64, since np.fft uses float64 internally + waveform = waveform.astype(np.float64) + window = window.astype(np.float64) + + # split waveform into frames of frame_length size + num_frames = int(1 + np.floor((waveform.size - frame_length) / hop_length)) + + num_frequency_bins = (fft_length // 2) + 1 if onesided else fft_length + spectrogram = np.empty((num_frames, num_frequency_bins), dtype=np.complex64) + + # rfft is faster than fft + fft_func = np.fft.rfft if onesided else np.fft.fft + buffer = np.zeros(fft_length) + + timestep = 0 + for frame_idx in range(num_frames): + buffer[:frame_length] = waveform[timestep : timestep + frame_length] + + if dither != 0.0: + buffer[:frame_length] += dither * np.random.randn(frame_length) + + if remove_dc_offset: + buffer[:frame_length] = buffer[:frame_length] - buffer[:frame_length].mean() + + if preemphasis is not None: + buffer[1:frame_length] -= preemphasis * buffer[: frame_length - 1] + buffer[0] *= 1 - preemphasis + + buffer[:frame_length] *= window + + spectrogram[frame_idx] = fft_func(buffer) + timestep += hop_length + + # note: ** is much faster than np.power + if power is not None: + spectrogram = np.abs(spectrogram, dtype=np.float64) ** power + + spectrogram = spectrogram.T + + if mel_filters is not None: + spectrogram = np.maximum(mel_floor, np.dot(mel_filters.T, spectrogram)) + + if power is not None and log_mel is not None: + if log_mel == "log": + spectrogram = np.log(spectrogram) + elif log_mel == "log10": + spectrogram = np.log10(spectrogram) + elif log_mel == "dB": + if power == 1.0: + spectrogram = amplitude_to_db(spectrogram, reference, min_value, db_range) + elif power == 2.0: + spectrogram = power_to_db(spectrogram, reference, min_value, db_range) + else: + raise ValueError(f"Cannot use log_mel option '{log_mel}' with power {power}") + else: + raise ValueError(f"Unknown log_mel option: {log_mel}") + + spectrogram = np.asarray(spectrogram, dtype) + + return spectrogram + + +def spectrogram_batch( + waveform_list: list[np.ndarray], + window: np.ndarray, + frame_length: int, + hop_length: int, + fft_length: Optional[int] = None, + power: Optional[float] = 1.0, + center: bool = True, + pad_mode: str = "reflect", + onesided: bool = True, + dither: float = 0.0, + preemphasis: Optional[float] = None, + mel_filters: Optional[np.ndarray] = None, + mel_floor: float = 1e-10, + log_mel: Optional[str] = None, + reference: float = 1.0, + min_value: float = 1e-10, + db_range: Optional[float] = None, + remove_dc_offset: Optional[bool] = None, + dtype: np.dtype = np.float32, +) -> list[np.ndarray]: + """ + Calculates spectrograms for a list of waveforms using the Short-Time Fourier Transform, optimized for batch processing. + This function extends the capabilities of the `spectrogram` function to handle multiple waveforms efficiently by leveraging broadcasting. + + It supports generating various types of spectrograms: + + - amplitude spectrogram (`power = 1.0`) + - power spectrogram (`power = 2.0`) + - complex-valued spectrogram (`power = None`) + - log spectrogram (use `log_mel` argument) + - mel spectrogram (provide `mel_filters`) + - log-mel spectrogram (provide `mel_filters` and `log_mel`) + + How this works: + + 1. The input waveform is split into frames of size `frame_length` that are partially overlapping by `frame_length + - hop_length` samples. + 2. Each frame is multiplied by the window and placed into a buffer of size `fft_length`. + 3. The DFT is taken of each windowed frame. + 4. The results are stacked into a spectrogram. + + We make a distinction between the following "blocks" of sample data, each of which may have a different lengths: + + - The analysis frame. This is the size of the time slices that the input waveform is split into. + - The window. Each analysis frame is multiplied by the window to avoid spectral leakage. + - The FFT input buffer. The length of this determines how many frequency bins are in the spectrogram. + + In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. A + padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, + typically the next power of two. + + Note: This function is designed for efficient batch processing of multiple waveforms but retains compatibility with individual waveform processing methods like `librosa.stft`. + + Args: + waveform_list (`List[np.ndarray]` with arrays of shape `(length,)`): + The list of input waveforms, each a single-channel (mono) signal. + window (`np.ndarray` of shape `(frame_length,)`): + The windowing function to apply, including zero-padding if necessary. + frame_length (`int`): + The length of each frame for analysis. + hop_length (`int`): + The step size between successive frames. + fft_length (`int`, *optional*): + The size of the FFT buffer, defining frequency bin resolution. + power (`float`, *optional*, defaults to 1.0): + Determines the type of spectrogram: 1.0 for amplitude, 2.0 for power, None for complex. + center (`bool`, *optional*, defaults to `True`): + Whether to center-pad the waveform frames. + pad_mode (`str`, *optional*, defaults to `"reflect"`): + The padding strategy when `center` is `True`. + onesided (`bool`, *optional*, defaults to `True`): + If True, returns a one-sided spectrogram for real input signals. + dither (`float`, *optional*, defaults to 0.0): + Adds dithering. In other words, adds a small Gaussian noise to each frame. + E.g. use 4.0 to add dithering with a normal distribution centered + around 0.0 with standard deviation 4.0, 0.0 means no dithering. + preemphasis (`float`, *optional*): + Applies a pre-emphasis filter to each frame. + mel_filters (`np.ndarray`, *optional*): + Mel filter bank for converting to mel spectrogram. + mel_floor (`float`, *optional*, defaults to 1e-10): + Floor value for mel spectrogram to avoid log(0). + log_mel (`str`, *optional*): + Specifies log scaling strategy; options are None, "log", "log10", "dB". + reference (`float`, *optional*, defaults to 1.0): + Reference value for dB conversion in log_mel. + min_value (`float`, *optional*, defaults to 1e-10): + Minimum floor value for log scale conversions. + db_range (`float`, *optional*): + Dynamic range for dB scale spectrograms. + remove_dc_offset (`bool`, *optional*): + Whether to remove the DC offset from each frame. + dtype (`np.dtype`, *optional*, defaults to `np.float32`): + Data type of the output spectrogram. + + Returns: + List[`np.ndarray`]: A list of spectrogram arrays, one for each input waveform. + """ + window_length = len(window) + + if fft_length is None: + fft_length = frame_length + + if frame_length > fft_length: + raise ValueError(f"frame_length ({frame_length}) may not be larger than fft_length ({fft_length})") + + if window_length != frame_length: + raise ValueError(f"Length of the window ({window_length}) must equal frame_length ({frame_length})") + + if hop_length <= 0: + raise ValueError("hop_length must be greater than zero") + + # Check the dimensions of the waveform , and if waveform is complex + for waveform in waveform_list: + if waveform.ndim != 1: + raise ValueError(f"Input waveform must have only one dimension, shape is {waveform.shape}") + if np.iscomplexobj(waveform): + raise ValueError("Complex-valued input waveforms are not currently supported") + # Center pad the waveform + if center: + padding = [(int(frame_length // 2), int(frame_length // 2))] + waveform_list = [ + np.pad( + waveform, + padding, + mode=pad_mode, + ) + for waveform in waveform_list + ] + original_waveform_lengths = [ + len(waveform) for waveform in waveform_list + ] # these lengths will be used to remove padding later + + # Batch pad the waveform + max_length = max(original_waveform_lengths) + padded_waveform_batch = np.array( + [ + np.pad(waveform, (0, max_length - len(waveform)), mode="constant", constant_values=0) + for waveform in waveform_list + ], + dtype=dtype, + ) + + # Promote to float64, since np.fft uses float64 internally + padded_waveform_batch = padded_waveform_batch.astype(np.float64) + window = window.astype(np.float64) + + # Split waveform into frames of frame_length size + num_frames = int(1 + np.floor((padded_waveform_batch.shape[1] - frame_length) / hop_length)) + # these lengths will be used to remove padding later + true_num_frames = [int(1 + np.floor((length - frame_length) / hop_length)) for length in original_waveform_lengths] + num_batches = padded_waveform_batch.shape[0] + + num_frequency_bins = (fft_length // 2) + 1 if onesided else fft_length + spectrogram = np.empty((num_batches, num_frames, num_frequency_bins), dtype=np.complex64) + + # rfft is faster than fft + fft_func = np.fft.rfft if onesided else np.fft.fft + buffer = np.zeros((num_batches, fft_length)) + + for frame_idx in range(num_frames): + timestep = frame_idx * hop_length + buffer[:, :frame_length] = padded_waveform_batch[:, timestep : timestep + frame_length] + + if dither != 0.0: + buffer[:, :frame_length] += dither * np.random.randn(*buffer[:, :frame_length].shape) + + if remove_dc_offset: + buffer[:, :frame_length] -= buffer[:, :frame_length].mean(axis=1, keepdims=True) + + if preemphasis is not None: + buffer[:, 1:frame_length] -= preemphasis * buffer[:, : frame_length - 1] + buffer[:, 0] *= 1 - preemphasis + + buffer[:, :frame_length] *= window + + spectrogram[:, frame_idx] = fft_func(buffer) + + # Note: ** is much faster than np.power + if power is not None: + spectrogram = np.abs(spectrogram, dtype=np.float64) ** power + + # Apply mel filters if provided + if mel_filters is not None: + result = np.tensordot(spectrogram, mel_filters.T, axes=([2], [1])) + spectrogram = np.maximum(mel_floor, result) + + # Convert to log scale if specified + if power is not None and log_mel is not None: + if log_mel == "log": + spectrogram = np.log(spectrogram) + elif log_mel == "log10": + spectrogram = np.log10(spectrogram) + elif log_mel == "dB": + if power == 1.0: + spectrogram = amplitude_to_db_batch(spectrogram, reference, min_value, db_range) + elif power == 2.0: + spectrogram = power_to_db_batch(spectrogram, reference, min_value, db_range) + else: + raise ValueError(f"Cannot use log_mel option '{log_mel}' with power {power}") + else: + raise ValueError(f"Unknown log_mel option: {log_mel}") + + spectrogram = np.asarray(spectrogram, dtype) + + spectrogram_list = [spectrogram[i, : true_num_frames[i], :].T for i in range(len(true_num_frames))] + + return spectrogram_list + + +def power_to_db( + spectrogram: np.ndarray, + reference: float = 1.0, + min_value: float = 1e-10, + db_range: Optional[float] = None, +) -> np.ndarray: + """ + Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`, using basic + logarithm properties for numerical stability. + + The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a + linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it. + This means that large variations in energy may not sound all that different if the sound is loud to begin with. + This compression operation makes the (mel) spectrogram features match more closely what humans actually hear. + + Based on the implementation of `librosa.power_to_db`. + + Args: + spectrogram (`np.ndarray`): + The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared! + reference (`float`, *optional*, defaults to 1.0): + Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set + the loudest part to 0 dB. Must be greater than zero. + min_value (`float`, *optional*, defaults to `1e-10`): + The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking + `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero. + db_range (`float`, *optional*): + Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the + peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + + Returns: + `np.ndarray`: the spectrogram in decibels + """ + if reference <= 0.0: + raise ValueError("reference must be greater than zero") + if min_value <= 0.0: + raise ValueError("min_value must be greater than zero") + + reference = max(min_value, reference) + + spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None) + spectrogram = 10.0 * (np.log10(spectrogram) - np.log10(reference)) + + if db_range is not None: + if db_range <= 0.0: + raise ValueError("db_range must be greater than zero") + spectrogram = np.clip(spectrogram, a_min=spectrogram.max() - db_range, a_max=None) + + return spectrogram + + +def power_to_db_batch( + spectrogram: np.ndarray, + reference: float = 1.0, + min_value: float = 1e-10, + db_range: Optional[float] = None, +) -> np.ndarray: + """ + Converts a batch of power spectrograms to the decibel scale. This computes `10 * log10(spectrogram / reference)`, + using basic logarithm properties for numerical stability. + + This function supports batch processing, where each item in the batch is an individual power (mel) spectrogram. + + Args: + spectrogram (`np.ndarray`): + The input batch of power (mel) spectrograms. Expected shape is (batch_size, *spectrogram_shape). + Note that a power spectrogram has the amplitudes squared! + reference (`float`, *optional*, defaults to 1.0): + Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set + the loudest part to 0 dB. Must be greater than zero. + min_value (`float`, *optional*, defaults to `1e-10`): + The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking + `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero. + db_range (`float`, *optional*): + Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the + peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + + Returns: + `np.ndarray`: the batch of spectrograms in decibels + """ + if reference <= 0.0: + raise ValueError("reference must be greater than zero") + if min_value <= 0.0: + raise ValueError("min_value must be greater than zero") + + reference = max(min_value, reference) + + spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None) + spectrogram = 10.0 * (np.log10(spectrogram) - np.log10(reference)) + + if db_range is not None: + if db_range <= 0.0: + raise ValueError("db_range must be greater than zero") + # Apply db_range clipping per batch item + max_values = spectrogram.max(axis=(1, 2), keepdims=True) + spectrogram = np.clip(spectrogram, a_min=max_values - db_range, a_max=None) + + return spectrogram + + +def amplitude_to_db( + spectrogram: np.ndarray, + reference: float = 1.0, + min_value: float = 1e-5, + db_range: Optional[float] = None, +) -> np.ndarray: + """ + Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`, using + basic logarithm properties for numerical stability. + + The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a + linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it. + This means that large variations in energy may not sound all that different if the sound is loud to begin with. + This compression operation makes the (mel) spectrogram features match more closely what humans actually hear. + + Args: + spectrogram (`np.ndarray`): + The input amplitude (mel) spectrogram. + reference (`float`, *optional*, defaults to 1.0): + Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set + the loudest part to 0 dB. Must be greater than zero. + min_value (`float`, *optional*, defaults to `1e-5`): + The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking + `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero. + db_range (`float`, *optional*): + Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the + peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + + Returns: + `np.ndarray`: the spectrogram in decibels + """ + if reference <= 0.0: + raise ValueError("reference must be greater than zero") + if min_value <= 0.0: + raise ValueError("min_value must be greater than zero") + + reference = max(min_value, reference) + + spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None) + spectrogram = 20.0 * (np.log10(spectrogram) - np.log10(reference)) + + if db_range is not None: + if db_range <= 0.0: + raise ValueError("db_range must be greater than zero") + spectrogram = np.clip(spectrogram, a_min=spectrogram.max() - db_range, a_max=None) + + return spectrogram + + +def amplitude_to_db_batch( + spectrogram: np.ndarray, reference: float = 1.0, min_value: float = 1e-5, db_range: Optional[float] = None +) -> np.ndarray: + """ + Converts a batch of amplitude spectrograms to the decibel scale. This computes `20 * log10(spectrogram / reference)`, + using basic logarithm properties for numerical stability. + + The function supports batch processing, where each item in the batch is an individual amplitude (mel) spectrogram. + + Args: + spectrogram (`np.ndarray`): + The input batch of amplitude (mel) spectrograms. Expected shape is (batch_size, *spectrogram_shape). + reference (`float`, *optional*, defaults to 1.0): + Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set + the loudest part to 0 dB. Must be greater than zero. + min_value (`float`, *optional*, defaults to `1e-5`): + The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking + `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero. + db_range (`float`, *optional*): + Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the + peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + + Returns: + `np.ndarray`: the batch of spectrograms in decibels + """ + if reference <= 0.0: + raise ValueError("reference must be greater than zero") + if min_value <= 0.0: + raise ValueError("min_value must be greater than zero") + + reference = max(min_value, reference) + + spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None) + spectrogram = 20.0 * (np.log10(spectrogram) - np.log10(reference)) + + if db_range is not None: + if db_range <= 0.0: + raise ValueError("db_range must be greater than zero") + # Apply db_range clipping per batch item + max_values = spectrogram.max(axis=(1, 2), keepdims=True) + spectrogram = np.clip(spectrogram, a_min=max_values - db_range, a_max=None) + + return spectrogram + + +### deprecated functions below this line ### + + +def get_mel_filter_banks( + nb_frequency_bins: int, + nb_mel_filters: int, + frequency_min: float, + frequency_max: float, + sample_rate: int, + norm: Optional[str] = None, + mel_scale: str = "htk", +) -> np.array: + warnings.warn( + "The function `get_mel_filter_banks` is deprecated and will be removed in version 4.31.0 of Transformers", + FutureWarning, + ) + return mel_filter_bank( + num_frequency_bins=nb_frequency_bins, + num_mel_filters=nb_mel_filters, + min_frequency=frequency_min, + max_frequency=frequency_max, + sampling_rate=sample_rate, + norm=norm, + mel_scale=mel_scale, + ) + + +def fram_wave(waveform: np.array, hop_length: int = 160, fft_window_size: int = 400, center: bool = True): + """ + In order to compute the short time fourier transform, the waveform needs to be split in overlapping windowed + segments called `frames`. + + The window length (window_length) defines how much of the signal is contained in each frame, while the hop length + defines the step between the beginning of each new frame. + + + Args: + waveform (`np.array` of shape `(sample_length,)`): + The raw waveform which will be split into smaller chunks. + hop_length (`int`, *optional*, defaults to 160): + Step between each window of the waveform. + fft_window_size (`int`, *optional*, defaults to 400): + Defines the size of the window. + center (`bool`, defaults to `True`): + Whether or not to center each frame around the middle of the frame. Centering is done by reflecting the + waveform on the left and on the right. + + Return: + framed_waveform (`np.array` of shape `(waveform.shape // hop_length , fft_window_size)`): + The framed waveforms that can be fed to `np.fft`. + """ + warnings.warn( + "The function `fram_wave` is deprecated and will be removed in version 4.31.0 of Transformers", + FutureWarning, + ) + frames = [] + for i in range(0, waveform.shape[0] + 1, hop_length): + if center: + half_window = (fft_window_size - 1) // 2 + 1 + start = i - half_window if i > half_window else 0 + end = i + half_window if i < waveform.shape[0] - half_window else waveform.shape[0] + frame = waveform[start:end] + if start == 0: + padd_width = (-i + half_window, 0) + frame = np.pad(frame, pad_width=padd_width, mode="reflect") + + elif end == waveform.shape[0]: + padd_width = (0, (i - waveform.shape[0] + half_window)) + frame = np.pad(frame, pad_width=padd_width, mode="reflect") + + else: + frame = waveform[i : i + fft_window_size] + frame_width = frame.shape[0] + if frame_width < waveform.shape[0]: + frame = np.lib.pad( + frame, pad_width=(0, fft_window_size - frame_width), mode="constant", constant_values=0 + ) + frames.append(frame) + + frames = np.stack(frames, 0) + return frames + + +def stft(frames: np.array, windowing_function: np.array, fft_window_size: Optional[int] = None): + """ + Calculates the complex Short-Time Fourier Transform (STFT) of the given framed signal. Should give the same results + as `torch.stft`. + + Args: + frames (`np.array` of dimension `(num_frames, fft_window_size)`): + A framed audio signal obtained using `audio_utils.fram_wav`. + windowing_function (`np.array` of dimension `(nb_frequency_bins, nb_mel_filters)`: + A array representing the function that will be used to reduces the amplitude of the discontinuities at the + boundaries of each frame when computing the STFT. Each frame will be multiplied by the windowing_function. + For more information on the discontinuities, called *Spectral leakage*, refer to [this + tutorial]https://download.ni.com/evaluation/pxi/Understanding%20FFTs%20and%20Windowing.pdf + fft_window_size (`int`, *optional*): + Size of the window om which the Fourier transform is applied. This controls the frequency resolution of the + spectrogram. 400 means that the fourrier transform is computed on windows of 400 samples. The number of + frequency bins (`nb_frequency_bins`) used to divide the window into equal strips is equal to + `(1+fft_window_size)//2`. An increase of the fft_window_size slows the calculus time proportionnally. + + Example: + + ```python + >>> from transformers.audio_utils import stft, fram_wave + >>> import numpy as np + + >>> audio = np.random.rand(50) + >>> fft_window_size = 10 + >>> hop_length = 2 + >>> framed_audio = fram_wave(audio, hop_length, fft_window_size) + >>> spectrogram = stft(framed_audio, np.hanning(fft_window_size + 1)) + ``` + + Returns: + spectrogram (`np.ndarray`): + A spectrogram of shape `(num_frames, nb_frequency_bins)` obtained using the STFT algorithm + """ + warnings.warn( + "The function `stft` is deprecated and will be removed in version 4.31.0 of Transformers", + FutureWarning, + ) + frame_size = frames.shape[1] + + if fft_window_size is None: + fft_window_size = frame_size + + if fft_window_size < frame_size: + raise ValueError("FFT size must greater or equal the frame size") + # number of FFT bins to store + nb_frequency_bins = (fft_window_size >> 1) + 1 + + spectrogram = np.empty((len(frames), nb_frequency_bins), dtype=np.complex64) + fft_signal = np.zeros(fft_window_size) + + for f, frame in enumerate(frames): + if windowing_function is not None: + np.multiply(frame, windowing_function, out=fft_signal[:frame_size]) + else: + fft_signal[:frame_size] = frame + spectrogram[f] = np.fft.fft(fft_signal, axis=0)[:nb_frequency_bins] + return spectrogram.T diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/auto.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/auto.py new file mode 100644 index 0000000000000000000000000000000000000000..9d24b3539530b5d1ae03972de776f1d610c703d9 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/auto.py @@ -0,0 +1,266 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# Modifications Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import warnings +from typing import Dict, Optional, Union + +from ..models.auto.configuration_auto import AutoConfig +from ..utils import logging +from ..utils.quantization_config import ( + AqlmConfig, + AwqConfig, + BitNetConfig, + BitsAndBytesConfig, + CompressedTensorsConfig, + EetqConfig, + FbgemmFp8Config, + FineGrainedFP8Config, + GPTQConfig, + HiggsConfig, + HqqConfig, + QuantizationConfigMixin, + QuantizationMethod, + QuantoConfig, + QuarkConfig, + SpQRConfig, + TorchAoConfig, + VptqConfig, +) +from .base import HfQuantizer +from .quantizer_aqlm import AqlmHfQuantizer +from .quantizer_awq import AwqQuantizer +from .quantizer_bitnet import BitNetHfQuantizer +from .quantizer_bnb_4bit import Bnb4BitHfQuantizer +from .quantizer_bnb_8bit import Bnb8BitHfQuantizer +from .quantizer_compressed_tensors import CompressedTensorsHfQuantizer +from .quantizer_eetq import EetqHfQuantizer +from .quantizer_fbgemm_fp8 import FbgemmFp8HfQuantizer +from .quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer +from .quantizer_gptq import GptqHfQuantizer +from .quantizer_higgs import HiggsHfQuantizer +from .quantizer_hqq import HqqHfQuantizer +from .quantizer_quanto import QuantoHfQuantizer +from .quantizer_quark import QuarkHfQuantizer +from .quantizer_spqr import SpQRHfQuantizer +from .quantizer_torchao import TorchAoHfQuantizer +from .quantizer_vptq import VptqHfQuantizer + + +AUTO_QUANTIZER_MAPPING = { + "awq": AwqQuantizer, + "bitsandbytes_4bit": Bnb4BitHfQuantizer, + "bitsandbytes_8bit": Bnb8BitHfQuantizer, + "gptq": GptqHfQuantizer, + "aqlm": AqlmHfQuantizer, + "quanto": QuantoHfQuantizer, + "quark": QuarkHfQuantizer, + "eetq": EetqHfQuantizer, + "higgs": HiggsHfQuantizer, + "hqq": HqqHfQuantizer, + "compressed-tensors": CompressedTensorsHfQuantizer, + "fbgemm_fp8": FbgemmFp8HfQuantizer, + "torchao": TorchAoHfQuantizer, + "bitnet": BitNetHfQuantizer, + "vptq": VptqHfQuantizer, + "spqr": SpQRHfQuantizer, + "fp8": FineGrainedFP8HfQuantizer, +} + +AUTO_QUANTIZATION_CONFIG_MAPPING = { + "awq": AwqConfig, + "bitsandbytes_4bit": BitsAndBytesConfig, + "bitsandbytes_8bit": BitsAndBytesConfig, + "eetq": EetqConfig, + "gptq": GPTQConfig, + "aqlm": AqlmConfig, + "quanto": QuantoConfig, + "quark": QuarkConfig, + "hqq": HqqConfig, + "compressed-tensors": CompressedTensorsConfig, + "fbgemm_fp8": FbgemmFp8Config, + "higgs": HiggsConfig, + "torchao": TorchAoConfig, + "bitnet": BitNetConfig, + "vptq": VptqConfig, + "spqr": SpQRConfig, + "fp8": FineGrainedFP8Config, +} + +logger = logging.get_logger(__name__) + + +class AutoQuantizationConfig: + """ + The Auto-HF quantization config class that takes care of automatically dispatching to the correct + quantization config given a quantization config stored in a dictionary. + """ + + @classmethod + def from_dict(cls, quantization_config_dict: Dict): + quant_method = quantization_config_dict.get("quant_method", None) + # We need a special care for bnb models to make sure everything is BC .. + if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False): + suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit" + quant_method = QuantizationMethod.BITS_AND_BYTES + suffix + elif quant_method is None: + raise ValueError( + "The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the model has been correctly quantized" + ) + + if quant_method not in AUTO_QUANTIZATION_CONFIG_MAPPING.keys(): + raise ValueError( + f"Unknown quantization type, got {quant_method} - supported types are:" + f" {list(AUTO_QUANTIZER_MAPPING.keys())}" + ) + + target_cls = AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method] + return target_cls.from_dict(quantization_config_dict) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + model_config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + if getattr(model_config, "quantization_config", None) is None: + raise ValueError( + f"Did not found a `quantization_config` in {pretrained_model_name_or_path}. Make sure that the model is correctly quantized." + ) + quantization_config_dict = model_config.quantization_config + quantization_config = cls.from_dict(quantization_config_dict) + # Update with potential kwargs that are passed through from_pretrained. + quantization_config.update(**kwargs) + return quantization_config + + +class AutoHfQuantizer: + """ + The Auto-HF quantizer class that takes care of automatically instantiating to the correct + `HfQuantizer` given the `QuantizationConfig`. + """ + + @classmethod + def from_config(cls, quantization_config: Union[QuantizationConfigMixin, Dict], **kwargs): + # Convert it to a QuantizationConfig if the q_config is a dict + if isinstance(quantization_config, dict): + quantization_config = AutoQuantizationConfig.from_dict(quantization_config) + + quant_method = quantization_config.quant_method + + # Again, we need a special care for bnb as we have a single quantization config + # class for both 4-bit and 8-bit quantization + if quant_method == QuantizationMethod.BITS_AND_BYTES: + if quantization_config.load_in_8bit: + quant_method += "_8bit" + else: + quant_method += "_4bit" + + if quant_method not in AUTO_QUANTIZER_MAPPING.keys(): + raise ValueError( + f"Unknown quantization type, got {quant_method} - supported types are:" + f" {list(AUTO_QUANTIZER_MAPPING.keys())}" + ) + + target_cls = AUTO_QUANTIZER_MAPPING[quant_method] + return target_cls(quantization_config, **kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + quantization_config = AutoQuantizationConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + return cls.from_config(quantization_config) + + @classmethod + def merge_quantization_configs( + cls, + quantization_config: Union[dict, QuantizationConfigMixin], + quantization_config_from_args: Optional[QuantizationConfigMixin], + ): + """ + handles situations where both quantization_config from args and quantization_config from model config are present. + """ + if quantization_config_from_args is not None: + warning_msg = ( + "You passed `quantization_config` or equivalent parameters to `from_pretrained` but the model you're loading" + " already has a `quantization_config` attribute. The `quantization_config` from the model will be used." + ) + else: + warning_msg = "" + + if isinstance(quantization_config, dict): + quantization_config = AutoQuantizationConfig.from_dict(quantization_config) + + if ( + isinstance(quantization_config, (GPTQConfig, AwqConfig, FbgemmFp8Config, CompressedTensorsConfig)) + and quantization_config_from_args is not None + ): + # special case for GPTQ / AWQ / FbgemmFp8 config collision + loading_attr_dict = quantization_config_from_args.get_loading_attributes() + for attr, val in loading_attr_dict.items(): + setattr(quantization_config, attr, val) + + warning_msg += f"However, loading attributes (e.g. {list(loading_attr_dict.keys())}) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored." + + if warning_msg != "": + warnings.warn(warning_msg) + + return quantization_config + + @staticmethod + def supports_quant_method(quantization_config_dict): + quant_method = quantization_config_dict.get("quant_method", None) + if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False): + suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit" + quant_method = QuantizationMethod.BITS_AND_BYTES + suffix + elif quant_method is None: + raise ValueError( + "The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the model has been correctly quantized" + ) + + if quant_method not in AUTO_QUANTIZATION_CONFIG_MAPPING.keys(): + logger.warning( + f"Unknown quantization type, got {quant_method} - supported types are:" + f" {list(AUTO_QUANTIZER_MAPPING.keys())}. Hence, we will skip the quantization. " + "To remove the warning, you can delete the quantization_config attribute in config.json" + ) + return False + return True + + +def register_quantization_config(method: str): + """Register a custom quantization configuration.""" + + def register_config_fn(cls): + if method in AUTO_QUANTIZATION_CONFIG_MAPPING: + raise ValueError(f"Config '{method}' already registered") + + if not issubclass(cls, QuantizationConfigMixin): + raise ValueError("Config must extend QuantizationConfigMixin") + + AUTO_QUANTIZATION_CONFIG_MAPPING[method] = cls + return cls + + return register_config_fn + + +def register_quantizer(name: str): + """Register a custom quantizer.""" + + def register_quantizer_fn(cls): + if name in AUTO_QUANTIZER_MAPPING: + raise ValueError(f"Quantizer '{name}' already registered") + + if not issubclass(cls, HfQuantizer): + raise ValueError("Quantizer must extend HfQuantizer") + + AUTO_QUANTIZER_MAPPING[name] = cls + return cls + + return register_quantizer_fn diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/auto_factory.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/auto_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..d21a16beb124c85e92182f569196be07a3d21a85 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/auto_factory.py @@ -0,0 +1,842 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Factory function to build auto-model classes.""" + +import copy +import importlib +import json +import warnings +from collections import OrderedDict + +from ...configuration_utils import PretrainedConfig +from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code +from ...utils import ( + CONFIG_NAME, + cached_file, + copy_func, + extract_commit_hash, + find_adapter_config_file, + is_peft_available, + is_torch_available, + logging, + requires_backends, +) +from .configuration_auto import AutoConfig, model_type_to_module_name, replace_list_option_in_docstrings + + +if is_torch_available(): + from ...generation import GenerationMixin + + +logger = logging.get_logger(__name__) + + +CLASS_DOCSTRING = """ + This is a generic model class that will be instantiated as one of the model classes of the library when created + with the [`~BaseAutoModelClass.from_pretrained`] class method or the [`~BaseAutoModelClass.from_config`] class + method. + + This class cannot be instantiated directly using `__init__()` (throws an error). +""" + +FROM_CONFIG_DOCSTRING = """ + Instantiates one of the model classes of the library from a configuration. + + Note: + Loading a model from its configuration file does **not** load the model weights. It only affects the + model's configuration. Use [`~BaseAutoModelClass.from_pretrained`] to load the model weights. + + Args: + config ([`PretrainedConfig`]): + The model class to instantiate is selected based on the configuration class: + + List options + attn_implementation (`str`, *optional*): + The attention implementation to use in the model (if relevant). Can be any of `"eager"` (manual implementation of the attention), `"sdpa"` (using [`F.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)), or `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)). By default, if available, SDPA will be used for torch>=2.1.1. The default is otherwise the manual `"eager"` implementation. + + Examples: + + ```python + >>> from transformers import AutoConfig, BaseAutoModelClass + + >>> # Download configuration from huggingface.co and cache. + >>> config = AutoConfig.from_pretrained("checkpoint_placeholder") + >>> model = BaseAutoModelClass.from_config(config) + ``` +""" + +FROM_PRETRAINED_TORCH_DOCSTRING = """ + Instantiate one of the model classes of the library from a pretrained model. + + The model class to instantiate is selected based on the `model_type` property of the config object (either + passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by + falling back to using pattern matching on `pretrained_model_name_or_path`: + + List options + + The model is set in evaluation mode by default using `model.eval()` (so for instance, dropout modules are + deactivated). To train the model, you should first set it back in training mode with `model.train()` + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In + this case, `from_tf` should be set to `True` and a configuration object should be provided as + `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a + PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + model_args (additional positional arguments, *optional*): + Will be passed along to the underlying model `__init__()` method. + config ([`PretrainedConfig`], *optional*): + Configuration for the model to use instead of an automatically loaded configuration. Configuration can + be automatically loaded when: + + - The model is a model provided by the library (loaded with the *model id* string of a pretrained + model). + - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the + save directory. + - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a + configuration JSON file named *config.json* is found in the directory. + state_dict (*Dict[str, torch.Tensor]*, *optional*): + A state dictionary to use instead of a state dictionary loaded from saved weights file. + + This option can be used if you want to create a model from a pretrained configuration but load your own + weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and + [`~PreTrainedModel.from_pretrained`] is not a simpler option. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + from_tf (`bool`, *optional*, defaults to `False`): + Load the model weights from a TensorFlow checkpoint save file (see docstring of + `pretrained_model_name_or_path` argument). + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (e.g., not try downloading the model). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + trust_remote_code (`bool`, *optional*, defaults to `False`): + Whether or not to allow for custom models defined on the Hub in their own modeling files. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + code_revision (`str`, *optional*, defaults to `"main"`): + The specific revision to use for the code on the Hub, if the code leaves in a different repository than + the rest of the model. It can be a branch name, a tag name, or a commit id, since we use a git-based + system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier + allowed by git. + kwargs (additional keyword arguments, *optional*): + Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., + `output_attentions=True`). Behaves differently depending on whether a `config` is provided or + automatically loaded: + + - If a configuration is provided with `config`, `**kwargs` will be directly passed to the + underlying model's `__init__` method (we assume all relevant updates to the configuration have + already been done) + - If a configuration is not provided, `kwargs` will be first passed to the configuration class + initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that + corresponds to a configuration attribute will be used to override said attribute with the + supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute + will be passed to the underlying model's `__init__` function. + + Examples: + + ```python + >>> from transformers import AutoConfig, BaseAutoModelClass + + >>> # Download model and configuration from huggingface.co and cache. + >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder") + + >>> # Update configuration during loading + >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder", output_attentions=True) + >>> model.config.output_attentions + True + + >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower) + >>> config = AutoConfig.from_pretrained("./tf_model/shortcut_placeholder_tf_model_config.json") + >>> model = BaseAutoModelClass.from_pretrained( + ... "./tf_model/shortcut_placeholder_tf_checkpoint.ckpt.index", from_tf=True, config=config + ... ) + ``` +""" + +FROM_PRETRAINED_TF_DOCSTRING = """ + Instantiate one of the model classes of the library from a pretrained model. + + The model class to instantiate is selected based on the `model_type` property of the config object (either + passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by + falling back to using pattern matching on `pretrained_model_name_or_path`: + + List options + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In this + case, `from_pt` should be set to `True` and a configuration object should be provided as `config` + argument. This loading path is slower than converting the PyTorch model in a TensorFlow model + using the provided conversion scripts and loading the TensorFlow model afterwards. + model_args (additional positional arguments, *optional*): + Will be passed along to the underlying model `__init__()` method. + config ([`PretrainedConfig`], *optional*): + Configuration for the model to use instead of an automatically loaded configuration. Configuration can + be automatically loaded when: + + - The model is a model provided by the library (loaded with the *model id* string of a pretrained + model). + - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the + save directory. + - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a + configuration JSON file named *config.json* is found in the directory. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + from_pt (`bool`, *optional*, defaults to `False`): + Load the model weights from a PyTorch checkpoint save file (see docstring of + `pretrained_model_name_or_path` argument). + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (e.g., not try downloading the model). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + trust_remote_code (`bool`, *optional*, defaults to `False`): + Whether or not to allow for custom models defined on the Hub in their own modeling files. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + code_revision (`str`, *optional*, defaults to `"main"`): + The specific revision to use for the code on the Hub, if the code leaves in a different repository than + the rest of the model. It can be a branch name, a tag name, or a commit id, since we use a git-based + system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier + allowed by git. + kwargs (additional keyword arguments, *optional*): + Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., + `output_attentions=True`). Behaves differently depending on whether a `config` is provided or + automatically loaded: + + - If a configuration is provided with `config`, `**kwargs` will be directly passed to the + underlying model's `__init__` method (we assume all relevant updates to the configuration have + already been done) + - If a configuration is not provided, `kwargs` will be first passed to the configuration class + initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that + corresponds to a configuration attribute will be used to override said attribute with the + supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute + will be passed to the underlying model's `__init__` function. + + Examples: + + ```python + >>> from transformers import AutoConfig, BaseAutoModelClass + + >>> # Download model and configuration from huggingface.co and cache. + >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder") + + >>> # Update configuration during loading + >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder", output_attentions=True) + >>> model.config.output_attentions + True + + >>> # Loading from a PyTorch checkpoint file instead of a TensorFlow model (slower) + >>> config = AutoConfig.from_pretrained("./pt_model/shortcut_placeholder_pt_model_config.json") + >>> model = BaseAutoModelClass.from_pretrained( + ... "./pt_model/shortcut_placeholder_pytorch_model.bin", from_pt=True, config=config + ... ) + ``` +""" + +FROM_PRETRAINED_FLAX_DOCSTRING = """ + Instantiate one of the model classes of the library from a pretrained model. + + The model class to instantiate is selected based on the `model_type` property of the config object (either + passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by + falling back to using pattern matching on `pretrained_model_name_or_path`: + + List options + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In this + case, `from_pt` should be set to `True` and a configuration object should be provided as `config` + argument. This loading path is slower than converting the PyTorch model in a TensorFlow model + using the provided conversion scripts and loading the TensorFlow model afterwards. + model_args (additional positional arguments, *optional*): + Will be passed along to the underlying model `__init__()` method. + config ([`PretrainedConfig`], *optional*): + Configuration for the model to use instead of an automatically loaded configuration. Configuration can + be automatically loaded when: + + - The model is a model provided by the library (loaded with the *model id* string of a pretrained + model). + - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the + save directory. + - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a + configuration JSON file named *config.json* is found in the directory. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + from_pt (`bool`, *optional*, defaults to `False`): + Load the model weights from a PyTorch checkpoint save file (see docstring of + `pretrained_model_name_or_path` argument). + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (e.g., not try downloading the model). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + trust_remote_code (`bool`, *optional*, defaults to `False`): + Whether or not to allow for custom models defined on the Hub in their own modeling files. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + code_revision (`str`, *optional*, defaults to `"main"`): + The specific revision to use for the code on the Hub, if the code leaves in a different repository than + the rest of the model. It can be a branch name, a tag name, or a commit id, since we use a git-based + system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier + allowed by git. + kwargs (additional keyword arguments, *optional*): + Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., + `output_attentions=True`). Behaves differently depending on whether a `config` is provided or + automatically loaded: + + - If a configuration is provided with `config`, `**kwargs` will be directly passed to the + underlying model's `__init__` method (we assume all relevant updates to the configuration have + already been done) + - If a configuration is not provided, `kwargs` will be first passed to the configuration class + initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that + corresponds to a configuration attribute will be used to override said attribute with the + supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute + will be passed to the underlying model's `__init__` function. + + Examples: + + ```python + >>> from transformers import AutoConfig, BaseAutoModelClass + + >>> # Download model and configuration from huggingface.co and cache. + >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder") + + >>> # Update configuration during loading + >>> model = BaseAutoModelClass.from_pretrained("checkpoint_placeholder", output_attentions=True) + >>> model.config.output_attentions + True + + >>> # Loading from a PyTorch checkpoint file instead of a TensorFlow model (slower) + >>> config = AutoConfig.from_pretrained("./pt_model/shortcut_placeholder_pt_model_config.json") + >>> model = BaseAutoModelClass.from_pretrained( + ... "./pt_model/shortcut_placeholder_pytorch_model.bin", from_pt=True, config=config + ... ) + ``` +""" + + +def _get_model_class(config, model_mapping): + supported_models = model_mapping[type(config)] + if not isinstance(supported_models, (list, tuple)): + return supported_models + + name_to_model = {model.__name__: model for model in supported_models} + architectures = getattr(config, "architectures", []) + for arch in architectures: + if arch in name_to_model: + return name_to_model[arch] + elif f"TF{arch}" in name_to_model: + return name_to_model[f"TF{arch}"] + elif f"Flax{arch}" in name_to_model: + return name_to_model[f"Flax{arch}"] + + # If not architecture is set in the config or match the supported models, the first element of the tuple is the + # defaults. + return supported_models[0] + + +class _BaseAutoModelClass: + # Base class for auto models. + _model_mapping = None + + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_config(config)` methods." + ) + + @classmethod + def from_config(cls, config, **kwargs): + trust_remote_code = kwargs.pop("trust_remote_code", None) + has_remote_code = hasattr(config, "auto_map") and cls.__name__ in config.auto_map + has_local_code = type(config) in cls._model_mapping.keys() + trust_remote_code = resolve_trust_remote_code( + trust_remote_code, config._name_or_path, has_local_code, has_remote_code + ) + + if has_remote_code and trust_remote_code: + class_ref = config.auto_map[cls.__name__] + if "--" in class_ref: + repo_id, class_ref = class_ref.split("--") + else: + repo_id = config.name_or_path + model_class = get_class_from_dynamic_module(class_ref, repo_id, **kwargs) + cls.register(config.__class__, model_class, exist_ok=True) + _ = kwargs.pop("code_revision", None) + model_class = add_generation_mixin_to_remote_model(model_class) + return model_class._from_config(config, **kwargs) + elif type(config) in cls._model_mapping.keys(): + model_class = _get_model_class(config, cls._model_mapping) + return model_class._from_config(config, **kwargs) + + raise ValueError( + f"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\n" + f"Model type should be one of {', '.join(c.__name__ for c in cls._model_mapping.keys())}." + ) + + @classmethod + def _prepare_config_for_auto_class(cls, config: PretrainedConfig) -> PretrainedConfig: + """Additional autoclass-specific config post-loading manipulation. May be overridden in subclasses.""" + return config + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + config = kwargs.pop("config", None) + trust_remote_code = kwargs.pop("trust_remote_code", None) + kwargs["_from_auto"] = True + hub_kwargs_names = [ + "cache_dir", + "force_download", + "local_files_only", + "proxies", + "resume_download", + "revision", + "subfolder", + "use_auth_token", + "token", + ] + hub_kwargs = {name: kwargs.pop(name) for name in hub_kwargs_names if name in kwargs} + code_revision = kwargs.pop("code_revision", None) + commit_hash = kwargs.pop("_commit_hash", None) + adapter_kwargs = kwargs.pop("adapter_kwargs", None) + + token = hub_kwargs.pop("token", None) + use_auth_token = hub_kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + hub_kwargs["token"] = token + + if commit_hash is None: + if not isinstance(config, PretrainedConfig): + # We make a call to the config file first (which may be absent) to get the commit hash as soon as possible + resolved_config_file = cached_file( + pretrained_model_name_or_path, + CONFIG_NAME, + _raise_exceptions_for_gated_repo=False, + _raise_exceptions_for_missing_entries=False, + _raise_exceptions_for_connection_errors=False, + **hub_kwargs, + ) + commit_hash = extract_commit_hash(resolved_config_file, commit_hash) + else: + commit_hash = getattr(config, "_commit_hash", None) + + if is_peft_available(): + if adapter_kwargs is None: + adapter_kwargs = {} + if token is not None: + adapter_kwargs["token"] = token + + maybe_adapter_path = find_adapter_config_file( + pretrained_model_name_or_path, _commit_hash=commit_hash, **adapter_kwargs + ) + + if maybe_adapter_path is not None: + with open(maybe_adapter_path, "r", encoding="utf-8") as f: + adapter_config = json.load(f) + + adapter_kwargs["_adapter_model_path"] = pretrained_model_name_or_path + pretrained_model_name_or_path = adapter_config["base_model_name_or_path"] + + if not isinstance(config, PretrainedConfig): + kwargs_orig = copy.deepcopy(kwargs) + # ensure not to pollute the config object with torch_dtype="auto" - since it's + # meaningless in the context of the config object - torch.dtype values are acceptable + if kwargs.get("torch_dtype", None) == "auto": + _ = kwargs.pop("torch_dtype") + # to not overwrite the quantization_config if config has a quantization_config + if kwargs.get("quantization_config", None) is not None: + _ = kwargs.pop("quantization_config") + + config, kwargs = AutoConfig.from_pretrained( + pretrained_model_name_or_path, + return_unused_kwargs=True, + trust_remote_code=trust_remote_code, + code_revision=code_revision, + _commit_hash=commit_hash, + **hub_kwargs, + **kwargs, + ) + + # if torch_dtype=auto was passed here, ensure to pass it on + if kwargs_orig.get("torch_dtype", None) == "auto": + kwargs["torch_dtype"] = "auto" + if kwargs_orig.get("quantization_config", None) is not None: + kwargs["quantization_config"] = kwargs_orig["quantization_config"] + + has_remote_code = hasattr(config, "auto_map") and cls.__name__ in config.auto_map + has_local_code = type(config) in cls._model_mapping.keys() + trust_remote_code = resolve_trust_remote_code( + trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code + ) + + # Set the adapter kwargs + kwargs["adapter_kwargs"] = adapter_kwargs + + if has_remote_code and trust_remote_code: + class_ref = config.auto_map[cls.__name__] + model_class = get_class_from_dynamic_module( + class_ref, pretrained_model_name_or_path, code_revision=code_revision, **hub_kwargs, **kwargs + ) + _ = hub_kwargs.pop("code_revision", None) + cls.register(config.__class__, model_class, exist_ok=True) + model_class = add_generation_mixin_to_remote_model(model_class) + return model_class.from_pretrained( + pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs + ) + elif type(config) in cls._model_mapping.keys(): + model_class = _get_model_class(config, cls._model_mapping) + if model_class.config_class == config.sub_configs.get("text_config", None): + config = config.get_text_config() + return model_class.from_pretrained( + pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs + ) + raise ValueError( + f"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\n" + f"Model type should be one of {', '.join(c.__name__ for c in cls._model_mapping.keys())}." + ) + + @classmethod + def register(cls, config_class, model_class, exist_ok=False): + """ + Register a new model for this class. + + Args: + config_class ([`PretrainedConfig`]): + The configuration corresponding to the model to register. + model_class ([`PreTrainedModel`]): + The model to register. + """ + if hasattr(model_class, "config_class") and model_class.config_class.__name__ != config_class.__name__: + raise ValueError( + "The model class you are passing has a `config_class` attribute that is not consistent with the " + f"config class you passed (model has {model_class.config_class} and you passed {config_class}. Fix " + "one of those so they match!" + ) + cls._model_mapping.register(config_class, model_class, exist_ok=exist_ok) + + +class _BaseAutoBackboneClass(_BaseAutoModelClass): + # Base class for auto backbone models. + _model_mapping = None + + @classmethod + def _load_timm_backbone_from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + requires_backends(cls, ["vision", "timm"]) + from ...models.timm_backbone import TimmBackboneConfig + + config = kwargs.pop("config", TimmBackboneConfig()) + + if kwargs.get("out_features", None) is not None: + raise ValueError("Cannot specify `out_features` for timm backbones") + + if kwargs.get("output_loading_info", False): + raise ValueError("Cannot specify `output_loading_info=True` when loading from timm") + + num_channels = kwargs.pop("num_channels", config.num_channels) + features_only = kwargs.pop("features_only", config.features_only) + use_pretrained_backbone = kwargs.pop("use_pretrained_backbone", config.use_pretrained_backbone) + out_indices = kwargs.pop("out_indices", config.out_indices) + config = TimmBackboneConfig( + backbone=pretrained_model_name_or_path, + num_channels=num_channels, + features_only=features_only, + use_pretrained_backbone=use_pretrained_backbone, + out_indices=out_indices, + ) + return super().from_config(config, **kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + use_timm_backbone = kwargs.pop("use_timm_backbone", False) + if use_timm_backbone: + return cls._load_timm_backbone_from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + + return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + + +def insert_head_doc(docstring, head_doc=""): + if len(head_doc) > 0: + return docstring.replace( + "one of the model classes of the library ", + f"one of the model classes of the library (with a {head_doc} head) ", + ) + return docstring.replace( + "one of the model classes of the library ", "one of the base model classes of the library " + ) + + +def auto_class_update(cls, checkpoint_for_example="google-bert/bert-base-cased", head_doc=""): + # Create a new class with the right name from the base class + model_mapping = cls._model_mapping + name = cls.__name__ + class_docstring = insert_head_doc(CLASS_DOCSTRING, head_doc=head_doc) + cls.__doc__ = class_docstring.replace("BaseAutoModelClass", name) + + # Now we need to copy and re-register `from_config` and `from_pretrained` as class methods otherwise we can't + # have a specific docstrings for them. + from_config = copy_func(_BaseAutoModelClass.from_config) + from_config_docstring = insert_head_doc(FROM_CONFIG_DOCSTRING, head_doc=head_doc) + from_config_docstring = from_config_docstring.replace("BaseAutoModelClass", name) + from_config_docstring = from_config_docstring.replace("checkpoint_placeholder", checkpoint_for_example) + from_config.__doc__ = from_config_docstring + from_config = replace_list_option_in_docstrings(model_mapping._model_mapping, use_model_types=False)(from_config) + cls.from_config = classmethod(from_config) + + if name.startswith("TF"): + from_pretrained_docstring = FROM_PRETRAINED_TF_DOCSTRING + elif name.startswith("Flax"): + from_pretrained_docstring = FROM_PRETRAINED_FLAX_DOCSTRING + else: + from_pretrained_docstring = FROM_PRETRAINED_TORCH_DOCSTRING + from_pretrained = copy_func(_BaseAutoModelClass.from_pretrained) + from_pretrained_docstring = insert_head_doc(from_pretrained_docstring, head_doc=head_doc) + from_pretrained_docstring = from_pretrained_docstring.replace("BaseAutoModelClass", name) + from_pretrained_docstring = from_pretrained_docstring.replace("checkpoint_placeholder", checkpoint_for_example) + shortcut = checkpoint_for_example.split("/")[-1].split("-")[0] + from_pretrained_docstring = from_pretrained_docstring.replace("shortcut_placeholder", shortcut) + from_pretrained.__doc__ = from_pretrained_docstring + from_pretrained = replace_list_option_in_docstrings(model_mapping._model_mapping)(from_pretrained) + cls.from_pretrained = classmethod(from_pretrained) + return cls + + +def get_values(model_mapping): + result = [] + for model in model_mapping.values(): + if isinstance(model, (list, tuple)): + result += list(model) + else: + result.append(model) + + return result + + +def getattribute_from_module(module, attr): + if attr is None: + return None + if isinstance(attr, tuple): + return tuple(getattribute_from_module(module, a) for a in attr) + if hasattr(module, attr): + return getattr(module, attr) + # Some of the mappings have entries model_type -> object of another model type. In that case we try to grab the + # object at the top level. + transformers_module = importlib.import_module("transformers") + + if module != transformers_module: + try: + return getattribute_from_module(transformers_module, attr) + except ValueError: + raise ValueError(f"Could not find {attr} neither in {module} nor in {transformers_module}!") + else: + raise ValueError(f"Could not find {attr} in {transformers_module}!") + + +def add_generation_mixin_to_remote_model(model_class): + """ + Adds `GenerationMixin` to the inheritance of `model_class`, if `model_class` is a PyTorch model. + + This function is used for backwards compatibility purposes: in v4.45, we've started a deprecation cycle to make + `PreTrainedModel` stop inheriting from `GenerationMixin`. Without this function, older models dynamically loaded + from the Hub may not have the `generate` method after we remove the inheritance. + """ + # 1. If it is not a PT model (i.e. doesn't inherit Module), do nothing + if "torch.nn.modules.module.Module" not in str(model_class.__mro__): + return model_class + + # 2. If it already **directly** inherits from GenerationMixin, do nothing + if "GenerationMixin" in str(model_class.__bases__): + return model_class + + # 3. Prior to v4.45, we could detect whether a model was `generate`-compatible if it had its own `generate` and/or + # `prepare_inputs_for_generation` method. + has_custom_generate = "GenerationMixin" not in str(getattr(model_class, "generate")) + has_custom_prepare_inputs = "GenerationMixin" not in str(getattr(model_class, "prepare_inputs_for_generation")) + if has_custom_generate or has_custom_prepare_inputs: + model_class_with_generation_mixin = type( + model_class.__name__, (model_class, GenerationMixin), {**model_class.__dict__} + ) + return model_class_with_generation_mixin + return model_class + + +class _LazyAutoMapping(OrderedDict): + """ + " A mapping config to object (model or tokenizer for instance) that will load keys and values when it is accessed. + + Args: + - config_mapping: The map model type to config class + - model_mapping: The map model type to model (or tokenizer) class + """ + + def __init__(self, config_mapping, model_mapping): + self._config_mapping = config_mapping + self._reverse_config_mapping = {v: k for k, v in config_mapping.items()} + self._model_mapping = model_mapping + self._model_mapping._model_mapping = self + self._extra_content = {} + self._modules = {} + + def __len__(self): + common_keys = set(self._config_mapping.keys()).intersection(self._model_mapping.keys()) + return len(common_keys) + len(self._extra_content) + + def __getitem__(self, key): + if key in self._extra_content: + return self._extra_content[key] + model_type = self._reverse_config_mapping[key.__name__] + if model_type in self._model_mapping: + model_name = self._model_mapping[model_type] + return self._load_attr_from_module(model_type, model_name) + + # Maybe there was several model types associated with this config. + model_types = [k for k, v in self._config_mapping.items() if v == key.__name__] + for mtype in model_types: + if mtype in self._model_mapping: + model_name = self._model_mapping[mtype] + return self._load_attr_from_module(mtype, model_name) + raise KeyError(key) + + def _load_attr_from_module(self, model_type, attr): + module_name = model_type_to_module_name(model_type) + if module_name not in self._modules: + self._modules[module_name] = importlib.import_module(f".{module_name}", "transformers.models") + return getattribute_from_module(self._modules[module_name], attr) + + def keys(self): + mapping_keys = [ + self._load_attr_from_module(key, name) + for key, name in self._config_mapping.items() + if key in self._model_mapping.keys() + ] + return mapping_keys + list(self._extra_content.keys()) + + def get(self, key, default): + try: + return self.__getitem__(key) + except KeyError: + return default + + def __bool__(self): + return bool(self.keys()) + + def values(self): + mapping_values = [ + self._load_attr_from_module(key, name) + for key, name in self._model_mapping.items() + if key in self._config_mapping.keys() + ] + return mapping_values + list(self._extra_content.values()) + + def items(self): + mapping_items = [ + ( + self._load_attr_from_module(key, self._config_mapping[key]), + self._load_attr_from_module(key, self._model_mapping[key]), + ) + for key in self._model_mapping.keys() + if key in self._config_mapping.keys() + ] + return mapping_items + list(self._extra_content.items()) + + def __iter__(self): + return iter(self.keys()) + + def __contains__(self, item): + if item in self._extra_content: + return True + if not hasattr(item, "__name__") or item.__name__ not in self._reverse_config_mapping: + return False + model_type = self._reverse_config_mapping[item.__name__] + return model_type in self._model_mapping + + def register(self, key, value, exist_ok=False): + """ + Register a new model in this mapping. + """ + if hasattr(key, "__name__") and key.__name__ in self._reverse_config_mapping: + model_type = self._reverse_config_mapping[key.__name__] + if model_type in self._model_mapping.keys() and not exist_ok: + raise ValueError(f"'{key}' is already used by a Transformers model.") + + self._extra_content[key] = value diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/awq.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/awq.py new file mode 100644 index 0000000000000000000000000000000000000000..c860ea1f53744bba6a4d63bc73ee51a187fc92f8 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/awq.py @@ -0,0 +1,498 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"AWQ (Activation aware Weight Quantization) integration file" + +import importlib + +from packaging import version + +from ..activations import ACT2FN +from ..modeling_utils import PreTrainedModel +from ..utils import is_auto_awq_available, is_ipex_available, is_torch_available, logging +from ..utils.quantization_config import ( + AwqBackendPackingMethod, + AwqConfig, + AWQLinearVersion, + ExllamaVersion, +) + + +if is_torch_available(): + import torch + import torch.nn as nn + +logger = logging.get_logger(__name__) + +AWQ_FUSED_MAPPINGS = { + "mistral": { + "attention": ["q_proj", "k_proj", "v_proj", "o_proj"], + "mlp": ["gate_proj", "up_proj", "down_proj"], + "layernorm": ["input_layernorm", "post_attention_layernorm", "norm"], + "use_alibi": False, + }, + "mixtral": { + "attention": ["q_proj", "k_proj", "v_proj", "o_proj"], + "mlp": ["w1", "w3", "w2"], + "layernorm": ["input_layernorm", "post_attention_layernorm", "norm"], + "use_alibi": False, + "rope_theta": 1000000.0, + }, + "llama": { + "attention": ["q_proj", "k_proj", "v_proj", "o_proj"], + "mlp": ["gate_proj", "up_proj", "down_proj"], + "layernorm": ["input_layernorm", "post_attention_layernorm", "norm"], + "use_alibi": False, + }, + "llava": { + "attention": ["q_proj", "k_proj", "v_proj", "o_proj"], + "mlp": ["gate_proj", "up_proj", "down_proj"], + "layernorm": ["input_layernorm", "post_attention_layernorm", "norm"], + "use_alibi": False, + }, +} + +AWQ_SCALES_MAPPINGS = { + "starcoder2": {"act": "act", "layer_before_act": "c_fc"}, + "RefinedWebModel": {"act": "act", "layer_before_act": "dense_h_to_4h"}, + "falcon": {"act": "act", "layer_before_act": "dense_h_to_4h"}, + "mpt": {"act": "act", "layer_before_act": "up_proj"}, + "gptj": {"act": "act", "layer_before_act": "fc_in"}, + "gpt_neox": {"act": "act", "layer_before_act": "dense_h_to_4h"}, + "gpt_bigcode": {"act": "act", "layer_before_act": "c_fc"}, + "bloom": {"act": "gelu_impl", "layer_before_act": "dense_h_to_4h"}, +} + + +def replace_quantization_scales(model, model_type): + from awq.modules.act import ScaledActivation + + if model_type not in AWQ_SCALES_MAPPINGS: + return model + for name, module in model.named_children(): + act_name = AWQ_SCALES_MAPPINGS[model_type]["act"] + layer_before_act_name = AWQ_SCALES_MAPPINGS[model_type]["layer_before_act"] + if name == act_name and hasattr(model, layer_before_act_name): + layer_before_act = getattr(model, AWQ_SCALES_MAPPINGS[model_type]["layer_before_act"]) + size = layer_before_act.out_features + scale_like = torch.ones(size) + model._modules[name] = ScaledActivation(module, scale_like) + _ = replace_quantization_scales(module, model_type) + return model + + +def replace_with_awq_linear( + model, + modules_to_not_convert=None, + quantization_config=None, + current_key_name=None, + has_been_replaced=False, +) -> bool: + """ + Public method that recursively replaces the Linear layers of the given model with AWQ quantized layers. + `accelerate` is needed to use this method. Returns the converted model and a boolean that indicates if the + conversion has been successfull or not. + + During the module replacement, we also infer the backend to use through the `quantization_config` object. + + Args: + model (`torch.nn.Module`): + The model to convert, can be any `torch.nn.Module` instance. + quantization_config (`AwqConfig`): + The quantization config object that contains the quantization parameters. + modules_to_not_convert (`list`, *optional*): + A list of modules to not convert. If a module name is in the list (e.g. `lm_head`), it will not be + converted. + current_key_name (`list`, *optional*): + A list that contains the current key name. This is used for recursion and should not be passed by the user. + has_been_replaced (`bool`, *optional*): + A boolean that indicates if the conversion has been successful or not. This is used for recursion and + should not be passed by the user. + """ + if modules_to_not_convert is None: + modules_to_not_convert = [] + + backend = quantization_config.backend + + if not is_auto_awq_available(): + raise ValueError( + "AWQ (either `autoawq` or `llmawq`) is not available. Please install it with `pip install autoawq` or check out the installation guide in https://github.com/mit-han-lab/llm-awq" + ) + + if backend == AwqBackendPackingMethod.AUTOAWQ: + if quantization_config.version == AWQLinearVersion.GEMM: + from awq.modules.linear.gemm import WQLinear_GEMM + + target_cls = WQLinear_GEMM + elif quantization_config.version == AWQLinearVersion.GEMV: + from awq.modules.linear.gemv import WQLinear_GEMV + + target_cls = WQLinear_GEMV + elif quantization_config.version == AWQLinearVersion.EXLLAMA: + if quantization_config.exllama_config["version"] == ExllamaVersion.ONE: + from awq.modules.linear.exllama import WQLinear_Exllama + + target_cls = WQLinear_Exllama + elif quantization_config.exllama_config["version"] == ExllamaVersion.TWO: + from awq.modules.linear.exllamav2 import WQLinear_ExllamaV2 + + target_cls = WQLinear_ExllamaV2 + else: + raise ValueError(f"Unrecognized Exllama version: {quantization_config.exllama_config['version']}") + elif quantization_config.version == AWQLinearVersion.IPEX: + from awq.modules.linear.gemm_ipex import WQLinear_IPEX + + target_cls = WQLinear_IPEX + else: + raise ValueError(f"Unrecognized AWQ version: {quantization_config.version}") + else: + from awq.quantize.qmodule import WQLinear + + target_cls = WQLinear + + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + if isinstance(module, nn.Linear) and name not in modules_to_not_convert: + # Check if the current key is not in the `modules_to_not_convert` + if not any(key in ".".join(current_key_name) for key in modules_to_not_convert): + in_features = module.in_features + out_features = module.out_features + + model._modules[name] = target_cls( + w_bit=quantization_config.bits, + group_size=quantization_config.group_size, + in_features=in_features, + out_features=out_features, + bias=module.bias is not None, + dev=module.weight.device, + ) + has_been_replaced = True + + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) + if len(list(module.children())) > 0: + _, has_been_replaced = replace_with_awq_linear( + module, + modules_to_not_convert=modules_to_not_convert, + current_key_name=current_key_name, + quantization_config=quantization_config, + has_been_replaced=has_been_replaced, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def get_modules_to_fuse(model, quantization_config): + """ + Returns the fusing mapping given the quantization config and the model + + Args: + model (`~PreTrainedModel`): + The model to fuse - note this model should have been converted into AWQ format beforehand. + quantization_config (`~transformers.quantization_config.AWQConfig`): + The quantization configuration to use. + """ + if not isinstance(model, PreTrainedModel): + raise TypeError(f"The model should be an instance of `PreTrainedModel`, got {model.__class__.__name__}") + + # Always default to `quantization_config.modules_to_fuse` + if quantization_config.modules_to_fuse is not None: + current_fused_mapping = quantization_config.modules_to_fuse + current_fused_mapping["max_seq_len"] = quantization_config.fuse_max_seq_len + elif model.config.model_type in AWQ_FUSED_MAPPINGS: + current_fused_mapping = AWQ_FUSED_MAPPINGS[model.config.model_type] + + # Properly deal with the case where we have a multi-modal model as well (e.g. Llava) + config = model.config.get_text_config(decoder=True) + + # Handle hidden_size, num_attention_heads, num_key_value_heads on our own. + hidden_size = config.hidden_size + num_attention_heads = config.num_attention_heads + num_key_value_heads = getattr(config, "num_key_value_heads", num_attention_heads) + + # Fill `current_fused_mapping` with the expected values + current_fused_mapping["hidden_size"] = hidden_size + current_fused_mapping["num_attention_heads"] = num_attention_heads + current_fused_mapping["num_key_value_heads"] = num_key_value_heads + current_fused_mapping["max_seq_len"] = quantization_config.fuse_max_seq_len + else: + raise ValueError( + "Fusing mapping not found either on the quantization config or the supported `AWQ_FUSED_MAPPINGS`. Please pass a `fused_mapping` argument" + " in the `quantization_config` or raise an issue on transformers https://github.com/huggingface/transformers to add its support." + ) + return current_fused_mapping + + +def fuse_awq_modules(model, quantization_config): + """ + Optionally fuse some modules in the model to speedup inference. + + Args: + model (`~PreTrainedModel`): + The model to fuse - note this model should have been converted into AWQ format beforehand. + quantization_config (`Union[AwqConfig, dict]`): + The quantization configuration to use. + """ + # We need to convert it from dict in order to get an AwqConfig object + # otherwise the fields `backend` etc. will not be available + # https://github.com/huggingface/transformers/pull/27411#discussion_r1414044495 + if isinstance(quantization_config, dict): + quantization_config = AwqConfig.from_dict(quantization_config) + backend = quantization_config.backend + + modules_to_fuse = get_modules_to_fuse(model, quantization_config) + modules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", None) + + if backend == AwqBackendPackingMethod.AUTOAWQ: + from awq.modules.fused.attn import QuantAttentionFused + from awq.modules.fused.mlp import QuantFusedMLP + from awq.modules.fused.norm import FasterTransformerRMSNorm + else: + raise ValueError("Fusing is only supported for the AutoAWQ backend") + + fused_attention_modules = [] + + for name, module in model.named_modules(): + if modules_to_not_convert is not None: + if any(module_name_to_not_convert in name for module_name_to_not_convert in modules_to_not_convert): + continue + + # Replace layer norms + _fuse_awq_layernorm(modules_to_fuse["layernorm"], module, FasterTransformerRMSNorm) + + # Replace MLP layers if awq version is not ipex. + if quantization_config.version != "ipex": + _fuse_awq_mlp(model, name, modules_to_fuse["mlp"], module, QuantFusedMLP) + else: + logger.info("The IPEX version AWQ does not support fuse mlp for now.") + + # Replace attention layers + attention_has_been_fused = _fuse_awq_attention_layers( + model, module, modules_to_fuse, name, QuantAttentionFused + ) + + if attention_has_been_fused: + fused_attention_modules.append(name.split(".")[0]) + + # For AWQ fused + Llama we need to set `config._attn_implementation` = "custom" to avoid unexpected behavior and pass + # `None` attention mask to the fused attention modules as now the attention mask is dropped by our models and dealt + # by the `AttentionMaskConverter` module. + if len(fused_attention_modules) > 0: + for module_name, module in model.named_modules(): + if any( + module_name in fused_attention_modules for fused_attention_parent_module in fused_attention_modules + ): + if hasattr(module, "config") and hasattr(module.config, "_attn_implementation"): + module.config._attn_implementation = "custom" + return model + + +def _fuse_awq_layernorm(fuse_module_names, module, target_cls): + """ + Fuse the LayerNorm layers into a target class using autoawq + + Args: + fuse_module_names (`List[str]`): + The list of module names to fuse + module (`nn.Module`): + The pytorch parent module that has layernorm modules to fuse + target_cls (`~autoawq.FasterTransformerRMSNorm`): + The `FasterTransformerRMSNorm` class as it only supports that class + for now. + """ + for module_name in fuse_module_names: + if hasattr(module, module_name): + old_module = getattr(module, module_name) + module._modules[module_name] = target_cls( + old_module.weight, + old_module.variance_epsilon, + ).to(old_module.weight.device) + del old_module + + +def _fuse_awq_mlp(model, current_module_name, fuse_module_names, module, target_cls): + """ + Fuse the MLP layers into a target class using autoawq + + Args: + model (`~PreTrainedModel`): + The input pretrained model + current_module_name (`str`): + The current submodule name + fuse_module_names (`List[str]`): + The list of module names to fuse. For the MLP layers it has to be an array + of length 3 that consists of the 3 MLP layers in the order (gate (dense layer post-attention) / up / down layers) + module (`nn.Module`): + The pytorch parent module that has layernorm modules to fuse + target_cls (`~autoawq.QuantFusedMLP`): + The `QuantFusedMLP` class as it only supports that class + for now. + """ + if len(fuse_module_names) == 0: + return + + if hasattr(module, fuse_module_names[0]): + gate_proj = getattr(module, fuse_module_names[0]) + up_proj = getattr(module, fuse_module_names[1]) + down_proj = getattr(module, fuse_module_names[2]) + + previous_device = gate_proj.qweight.device + + # Deal also with the case model has `text_config` attribute + config = model.config.get_text_config(decoder=True) + hidden_act = config.hidden_act + activation_fn = ACT2FN[hidden_act] + new_module = target_cls(gate_proj, down_proj, up_proj, activation_fn) + + parent_name, child_name = current_module_name.rsplit(".", 1) + parent = model.get_submodule(parent_name) + setattr(parent, child_name, new_module.to(previous_device)) + + del gate_proj, up_proj, down_proj + + +def _fuse_awq_attention_layers(model, module, modules_to_fuse, current_module_name, target_cls): + """ + Fuse the Attention layers into a target class using autoawq + + Args: + model (`~PreTrainedModel`): + The input pretrained model + module (`nn.Module`): + The pytorch parent module that has layernorm modules to fuse + modules_to_fuse (`List[str]`): + The module fusing mapping. The dictionary has to contain a field `attention` with attention module names + in the correct order: q, k, v, o layer + current_module_name (`str`): + The current submodule name + target_cls (`~autoawq.QuantAttentionFused`): + The `QuantAttentionFused` class as it only supports that class + for now. + """ + from awq.modules.linear import WQLinear_GEMM, WQLinear_GEMV + + module_has_been_fused = False + + if len(modules_to_fuse["attention"]) == 0: + return module_has_been_fused + + if hasattr(module, modules_to_fuse["attention"][0]): + # First, we pack the QKV layers together + q_proj = getattr(module, modules_to_fuse["attention"][0]) + + if isinstance(q_proj, WQLinear_GEMV): + linear_target_cls = WQLinear_GEMV + cat_dim = 0 + elif isinstance(q_proj, WQLinear_GEMM): + linear_target_cls = WQLinear_GEMM + cat_dim = 1 + elif is_ipex_available() and version.parse(importlib.metadata.version("autoawq")) > version.parse("0.2.6"): + from awq.modules.linear import WQLinear_IPEX + + if isinstance(q_proj, WQLinear_IPEX): + linear_target_cls = WQLinear_IPEX + cat_dim = 1 + else: + raise ValueError("Unsupported q_proj type: {type(q_proj)}") + + previous_device = q_proj.qweight.device + + k_proj = getattr(module, modules_to_fuse["attention"][1]) + v_proj = getattr(module, modules_to_fuse["attention"][2]) + o_proj = getattr(module, modules_to_fuse["attention"][3]) + + bias = torch.cat([q_proj.bias, k_proj.bias, v_proj.bias], dim=0) if q_proj.bias is not None else None + + qkv_layer = linear_target_cls( + q_proj.w_bit, + q_proj.group_size, + q_proj.in_features, + q_proj.out_features + k_proj.out_features + v_proj.out_features, + q_proj.bias is not None, + next(iter(module.state_dict().values())).device, + ) + + qkv_layer.qweight = torch.cat([q_proj.qweight, k_proj.qweight, v_proj.qweight], dim=cat_dim) + qkv_layer.qzeros = torch.cat([q_proj.qzeros, k_proj.qzeros, v_proj.qzeros], dim=cat_dim) + qkv_layer.scales = torch.cat([q_proj.scales, k_proj.scales, v_proj.scales], dim=cat_dim) + + if isinstance(qkv_layer, WQLinear_GEMV): + qkv_layer.split_k_iters = q_proj.split_k_iters + + qkv_layer.bias = bias + + fused_attention_layer = target_cls( + modules_to_fuse["hidden_size"], + modules_to_fuse["num_attention_heads"], + modules_to_fuse["num_key_value_heads"], + qkv_layer, + o_proj, + previous_device, + modules_to_fuse["max_seq_len"], + use_alibi=modules_to_fuse["use_alibi"], + # The default value in autoawq is set to 10000.0 + rope_theta=modules_to_fuse.get("rope_theta", 10000.0), + ) + + fused_attention_layer.is_hf_transformers = True + + parent_name, child_name = current_module_name.rsplit(".", 1) + parent = model.get_submodule(parent_name) + setattr(parent, child_name, fused_attention_layer.to(previous_device)) + + del q_proj, k_proj, v_proj, o_proj + module_has_been_fused = True + + return module_has_been_fused + + +def post_init_awq_exllama_modules(model, exllama_config): + """ + Runs post init for Exllama layers which performs: + - Weights unpacking, reordering and repacking + - Devices scratch space allocation + """ + + if exllama_config["version"] == ExllamaVersion.ONE: + from awq.modules.linear.exllama import exllama_post_init + + model = exllama_post_init(model) + elif exllama_config["version"] == ExllamaVersion.TWO: + from awq.modules.linear.exllamav2 import exllamav2_post_init + + model = exllamav2_post_init( + model, + max_input_len=exllama_config["max_input_len"], + max_batch_size=exllama_config["max_batch_size"], + ) + else: + raise ValueError(f"Unrecognized Exllama version: {exllama_config['version']}") + + return model + + +def post_init_awq_ipex_modules(model): + """ + Runs post init for IPEX layers which performs: + - Weights packing, reordering and repacking + """ + + from awq.modules.linear.gemm_ipex import ipex_post_init + + model = ipex_post_init(model) + + return model diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/backbone_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/backbone_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a6ab0046bc5c77d9a18450bd7518df16c31faabd --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/backbone_utils.py @@ -0,0 +1,377 @@ +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of utils to be used by backbones and their components.""" + +import enum +import inspect +from collections.abc import Iterable +from typing import TYPE_CHECKING, Optional, Union + + +if TYPE_CHECKING: + from ..configuration_utils import PretrainedConfig + + +class BackboneType(enum.Enum): + TIMM = "timm" + TRANSFORMERS = "transformers" + + +def verify_out_features_out_indices( + out_features: Optional[Iterable[str]], out_indices: Optional[Iterable[int]], stage_names: Optional[Iterable[str]] +): + """ + Verify that out_indices and out_features are valid for the given stage_names. + """ + if stage_names is None: + raise ValueError("Stage_names must be set for transformers backbones") + + if out_features is not None: + if not isinstance(out_features, (list,)): + raise ValueError(f"out_features must be a list got {type(out_features)}") + if any(feat not in stage_names for feat in out_features): + raise ValueError(f"out_features must be a subset of stage_names: {stage_names} got {out_features}") + if len(out_features) != len(set(out_features)): + raise ValueError(f"out_features must not contain any duplicates, got {out_features}") + if out_features != (sorted_feats := [feat for feat in stage_names if feat in out_features]): + raise ValueError( + f"out_features must be in the same order as stage_names, expected {sorted_feats} got {out_features}" + ) + + if out_indices is not None: + if not isinstance(out_indices, list): + raise ValueError(f"out_indices must be a list, got {type(out_indices)}") + # Convert negative indices to their positive equivalent: [-1,] -> [len(stage_names) - 1,] + positive_indices = tuple(idx % len(stage_names) if idx < 0 else idx for idx in out_indices) + if any(idx for idx in positive_indices if idx not in range(len(stage_names))): + raise ValueError(f"out_indices must be valid indices for stage_names {stage_names}, got {out_indices}") + if len(positive_indices) != len(set(positive_indices)): + msg = f"out_indices must not contain any duplicates, got {out_indices}" + msg += f"(equivalent to {positive_indices}))" if positive_indices != out_indices else "" + raise ValueError(msg) + if positive_indices != tuple(sorted(positive_indices)): + sorted_negative = [idx for _, idx in sorted(zip(positive_indices, out_indices), key=lambda x: x[0])] + raise ValueError( + f"out_indices must be in the same order as stage_names, expected {sorted_negative} got {out_indices}" + ) + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + if out_features != [stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + +def _align_output_features_output_indices( + out_features: Optional[list[str]], + out_indices: Optional[Union[list[int], tuple[int]]], + stage_names: list[str], +): + """ + Finds the corresponding `out_features` and `out_indices` for the given `stage_names`. + + The logic is as follows: + - `out_features` not set, `out_indices` set: `out_features` is set to the `out_features` corresponding to the + `out_indices`. + - `out_indices` not set, `out_features` set: `out_indices` is set to the `out_indices` corresponding to the + `out_features`. + - `out_indices` and `out_features` not set: `out_indices` and `out_features` are set to the last stage. + - `out_indices` and `out_features` set: input `out_indices` and `out_features` are returned. + + Args: + out_features (`List[str]`): The names of the features for the backbone to output. + out_indices (`List[int]` or `Tuple[int]`): The indices of the features for the backbone to output. + stage_names (`List[str]`): The names of the stages of the backbone. + """ + if out_indices is None and out_features is None: + out_indices = [len(stage_names) - 1] + out_features = [stage_names[-1]] + elif out_indices is None and out_features is not None: + out_indices = [stage_names.index(layer) for layer in out_features] + elif out_features is None and out_indices is not None: + out_features = [stage_names[idx] for idx in out_indices] + return out_features, out_indices + + +def get_aligned_output_features_output_indices( + out_features: Optional[list[str]], + out_indices: Optional[Union[list[int], tuple[int]]], + stage_names: list[str], +) -> tuple[list[str], list[int]]: + """ + Get the `out_features` and `out_indices` so that they are aligned. + + The logic is as follows: + - `out_features` not set, `out_indices` set: `out_features` is set to the `out_features` corresponding to the + `out_indices`. + - `out_indices` not set, `out_features` set: `out_indices` is set to the `out_indices` corresponding to the + `out_features`. + - `out_indices` and `out_features` not set: `out_indices` and `out_features` are set to the last stage. + - `out_indices` and `out_features` set: they are verified to be aligned. + + Args: + out_features (`List[str]`): The names of the features for the backbone to output. + out_indices (`List[int]` or `Tuple[int]`): The indices of the features for the backbone to output. + stage_names (`List[str]`): The names of the stages of the backbone. + """ + out_indices = list(out_indices) if out_indices is not None else None + # First verify that the out_features and out_indices are valid + verify_out_features_out_indices(out_features=out_features, out_indices=out_indices, stage_names=stage_names) + output_features, output_indices = _align_output_features_output_indices( + out_features=out_features, out_indices=out_indices, stage_names=stage_names + ) + # Verify that the aligned out_features and out_indices are valid + verify_out_features_out_indices(out_features=output_features, out_indices=output_indices, stage_names=stage_names) + return output_features, output_indices + + +class BackboneMixin: + backbone_type: Optional[BackboneType] = None + + def _init_timm_backbone(self, config) -> None: + """ + Initialize the backbone model from timm The backbone must already be loaded to self._backbone + """ + if getattr(self, "_backbone", None) is None: + raise ValueError("self._backbone must be set before calling _init_timm_backbone") + + # These will diagree with the defaults for the transformers models e.g. for resnet50 + # the transformer model has out_features = ['stem', 'stage1', 'stage2', 'stage3', 'stage4'] + # the timm model has out_features = ['act', 'layer1', 'layer2', 'layer3', 'layer4'] + self.stage_names = [stage["module"] for stage in self._backbone.feature_info.info] + self.num_features = [stage["num_chs"] for stage in self._backbone.feature_info.info] + + # In some timm versions, out_indices reflects the input type of out_indices on the `create_model` call, + # in later versions >= 1, it is always a tuple + out_indices = list(self._backbone.feature_info.out_indices) + out_features = self._backbone.feature_info.module_name() + + # We verify the out indices and out features are valid + verify_out_features_out_indices( + out_features=out_features, out_indices=out_indices, stage_names=self.stage_names + ) + self._out_features, self._out_indices = out_features, out_indices + + def _init_transformers_backbone(self, config) -> None: + stage_names = getattr(config, "stage_names") + out_features = getattr(config, "out_features", None) + out_indices = getattr(config, "out_indices", None) + + self.stage_names = stage_names + self._out_features, self._out_indices = get_aligned_output_features_output_indices( + out_features=out_features, out_indices=out_indices, stage_names=stage_names + ) + # Number of channels for each stage. This is set in the transformer backbone model init + self.num_features = None + + def _init_backbone(self, config) -> None: + """ + Method to initialize the backbone. This method is called by the constructor of the base class after the + pretrained model weights have been loaded. + """ + self.config = config + + self.use_timm_backbone = getattr(config, "use_timm_backbone", False) + self.backbone_type = BackboneType.TIMM if self.use_timm_backbone else BackboneType.TRANSFORMERS + + if self.backbone_type == BackboneType.TIMM: + self._init_timm_backbone(config) + elif self.backbone_type == BackboneType.TRANSFORMERS: + self._init_transformers_backbone(config) + else: + raise ValueError(f"backbone_type {self.backbone_type} not supported.") + + @property + def out_features(self): + return self._out_features + + @out_features.setter + def out_features(self, out_features: list[str]): + """ + Set the out_features attribute. This will also update the out_indices attribute to match the new out_features. + """ + self._out_features, self._out_indices = get_aligned_output_features_output_indices( + out_features=out_features, out_indices=None, stage_names=self.stage_names + ) + + @property + def out_indices(self): + return self._out_indices + + @out_indices.setter + def out_indices(self, out_indices: Union[tuple[int], list[int]]): + """ + Set the out_indices attribute. This will also update the out_features attribute to match the new out_indices. + """ + self._out_features, self._out_indices = get_aligned_output_features_output_indices( + out_features=None, out_indices=out_indices, stage_names=self.stage_names + ) + + @property + def out_feature_channels(self): + # the current backbones will output the number of channels for each stage + # even if that stage is not in the out_features list. + return {stage: self.num_features[i] for i, stage in enumerate(self.stage_names)} + + @property + def channels(self): + return [self.out_feature_channels[name] for name in self.out_features] + + def forward_with_filtered_kwargs(self, *args, **kwargs): + signature = dict(inspect.signature(self.forward).parameters) + filtered_kwargs = {k: v for k, v in kwargs.items() if k in signature} + return self(*args, **filtered_kwargs) + + def forward( + self, + pixel_values, + output_hidden_states: Optional[bool] = None, + output_attentions: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + raise NotImplementedError("This method should be implemented by the derived class.") + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default `to_dict()` from `PretrainedConfig` to + include the `out_features` and `out_indices` attributes. + """ + output = super().to_dict() + output["out_features"] = output.pop("_out_features") + output["out_indices"] = output.pop("_out_indices") + return output + + +class BackboneConfigMixin: + """ + A Mixin to support handling the `out_features` and `out_indices` attributes for the backbone configurations. + """ + + @property + def out_features(self): + return self._out_features + + @out_features.setter + def out_features(self, out_features: list[str]): + """ + Set the out_features attribute. This will also update the out_indices attribute to match the new out_features. + """ + self._out_features, self._out_indices = get_aligned_output_features_output_indices( + out_features=out_features, out_indices=None, stage_names=self.stage_names + ) + + @property + def out_indices(self): + return self._out_indices + + @out_indices.setter + def out_indices(self, out_indices: Union[tuple[int], list[int]]): + """ + Set the out_indices attribute. This will also update the out_features attribute to match the new out_indices. + """ + self._out_features, self._out_indices = get_aligned_output_features_output_indices( + out_features=None, out_indices=out_indices, stage_names=self.stage_names + ) + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default `to_dict()` from `PretrainedConfig` to + include the `out_features` and `out_indices` attributes. + """ + output = super().to_dict() + output["out_features"] = output.pop("_out_features") + output["out_indices"] = output.pop("_out_indices") + return output + + +def load_backbone(config): + """ + Loads the backbone model from a config object. + + If the config is from the backbone model itself, then we return a backbone model with randomly initialized + weights. + + If the config is from the parent model of the backbone model itself, then we load the pretrained backbone weights + if specified. + """ + from transformers import AutoBackbone, AutoConfig + + backbone_config = getattr(config, "backbone_config", None) + use_timm_backbone = getattr(config, "use_timm_backbone", None) + use_pretrained_backbone = getattr(config, "use_pretrained_backbone", None) + backbone_checkpoint = getattr(config, "backbone", None) + backbone_kwargs = getattr(config, "backbone_kwargs", None) + backbone_kwargs = {} if backbone_kwargs is None else backbone_kwargs + + if backbone_kwargs and backbone_config is not None: + raise ValueError("You can't specify both `backbone_kwargs` and `backbone_config`.") + + # If there is a backbone_config and a backbone checkpoint, and use_pretrained_backbone=False then the desired + # behaviour is ill-defined: do you want to load from the checkpoint's config or the backbone_config? + if backbone_config is not None and backbone_checkpoint is not None and use_pretrained_backbone is not None: + raise ValueError("Cannot specify both config.backbone_config and config.backbone") + + # If any of thhe following are set, then the config passed in is from a model which contains a backbone. + if ( + backbone_config is None + and use_timm_backbone is None + and backbone_checkpoint is None + and backbone_checkpoint is None + ): + return AutoBackbone.from_config(config=config, **backbone_kwargs) + + # config from the parent model that has a backbone + if use_timm_backbone: + if backbone_checkpoint is None: + raise ValueError("config.backbone must be set if use_timm_backbone is True") + # Because of how timm backbones were originally added to models, we need to pass in use_pretrained_backbone + # to determine whether to load the pretrained weights. + backbone = AutoBackbone.from_pretrained( + backbone_checkpoint, + use_timm_backbone=use_timm_backbone, + use_pretrained_backbone=use_pretrained_backbone, + **backbone_kwargs, + ) + elif use_pretrained_backbone: + if backbone_checkpoint is None: + raise ValueError("config.backbone must be set if use_pretrained_backbone is True") + backbone = AutoBackbone.from_pretrained(backbone_checkpoint, **backbone_kwargs) + else: + if backbone_config is None and backbone_checkpoint is None: + raise ValueError("Either config.backbone_config or config.backbone must be set") + if backbone_config is None: + backbone_config = AutoConfig.from_pretrained(backbone_checkpoint, **backbone_kwargs) + backbone = AutoBackbone.from_config(config=backbone_config) + return backbone + + +def verify_backbone_config_arguments( + use_timm_backbone: bool, + use_pretrained_backbone: bool, + backbone: Optional[str], + backbone_config: Optional[Union[dict, "PretrainedConfig"]], + backbone_kwargs: Optional[dict], +): + """ + Verify that the config arguments to be passed to load_backbone are valid + """ + if backbone_config is not None and backbone is not None: + raise ValueError("You can't specify both `backbone` and `backbone_config`.") + + if backbone_config is not None and use_timm_backbone: + raise ValueError("You can't specify both `backbone_config` and `use_timm_backbone`.") + + if backbone_kwargs is not None and backbone_kwargs and backbone_config is not None: + raise ValueError("You can't specify both `backbone_kwargs` and `backbone_config`.") diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/base.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/base.py new file mode 100644 index 0000000000000000000000000000000000000000..f2af16ab83a1036dc029fa34a24d09cc515e82a0 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/base.py @@ -0,0 +1,349 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from ..utils import is_torch_available +from ..utils.quantization_config import QuantizationConfigMixin, QuantizationMethod +from .quantizers_utils import get_module_from_name + + +if TYPE_CHECKING: + from ..modeling_utils import PreTrainedModel + +if is_torch_available(): + import torch + from torch.nn import ModuleList +else: + ModuleList = str + + +class HfQuantizer(ABC): + """ + Abstract class of the HuggingFace quantizer. Supports for now quantizing HF transformers models for inference and/or quantization. + This class is used only for transformers.PreTrainedModel.from_pretrained and cannot be easily used outside the scope of that method + yet. + + Attributes + quantization_config (`transformers.utils.quantization_config.QuantizationConfigMixin`): + The quantization config that defines the quantization parameters of your model that you want to quantize. + modules_to_not_convert (`List[str]`, *optional*): + The list of module names to not convert when quantizing the model. + required_packages (`List[str]`, *optional*): + The list of required pip packages to install prior to using the quantizer + requires_calibration (`bool`): + Whether the quantization method requires to calibrate the model before using it. + requires_parameters_quantization (`bool`): + Whether the quantization method requires to create a new Parameter. For example, for bitsandbytes, it is + required to create a new xxxParameter in order to properly quantize the model. + """ + + requires_calibration = False + required_packages = None + requires_parameters_quantization = False + + def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs): + self.quantization_config = quantization_config + + # -- Handle extra kwargs below -- + self.modules_to_not_convert = kwargs.pop("modules_to_not_convert", []) + self.pre_quantized = kwargs.pop("pre_quantized", True) + + if not self.pre_quantized and self.requires_calibration: + raise ValueError( + f"The quantization method {quantization_config.quant_method} does require the model to be pre-quantized." + f" You explicitly passed `pre_quantized=False` meaning your model weights are not quantized. Make sure to " + f"pass `pre_quantized=True` while knowing what you are doing." + ) + + def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype": + """ + Some quantization methods require to explicitly set the dtype of the model to a + target dtype. You need to override this method in case you want to make sure that behavior is + preserved + + Args: + torch_dtype (`torch.dtype`): + The input dtype that is passed in `from_pretrained` + """ + return torch_dtype + + def update_device_map(self, device_map: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """ + Override this method if you want to pass a override the existing device map with a new + one. E.g. for bitsandbytes, since `accelerate` is a hard requirement, if no device_map is + passed, the device_map is set to `"auto"`` + + Args: + device_map (`Union[dict, str]`, *optional*): + The device_map that is passed through the `from_pretrained` method. + """ + return device_map + + def adjust_target_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype": + """ + Override this method if you want to adjust the `target_dtype` variable used in `from_pretrained` + to compute the device_map in case the device_map is a `str`. E.g. for bitsandbytes we force-set `target_dtype` + to `torch.int8` and for 4-bit we pass a custom enum `accelerate.CustomDtype.int4`. + + Args: + torch_dtype (`torch.dtype`, *optional*): + The torch_dtype that is used to compute the device_map. + """ + return torch_dtype + + def update_missing_keys(self, model, missing_keys: List[str], prefix: str) -> List[str]: + """ + Override this method if you want to adjust the `missing_keys`. + + Args: + missing_keys (`List[str]`, *optional*): + The list of missing keys in the checkpoint compared to the state dict of the model + """ + return missing_keys + + def update_unexpected_keys(self, model, unexpected_keys: List[str], prefix: str) -> List[str]: + """ + Override this method if you want to adjust the `unexpected_keys`. + + Args: + unexpected_keys (`List[str]`, *optional*): + The list of unexpected keys in the checkpoint compared to the state dict of the model + """ + return unexpected_keys + + def update_missing_keys_after_loading(self, model, missing_keys: List[str], prefix: str) -> List[str]: + """ + Override this method if you want to adjust the `missing_keys` after loading the model params, + but before the model is post-processed. + + Args: + missing_keys (`List[str]`, *optional*): + The list of missing keys in the checkpoint compared to the state dict of the model + """ + return missing_keys + + def update_expected_keys(self, model, expected_keys: List[str], loaded_keys: List[str]) -> List[str]: + """ + Override this method if you want to adjust the `update_expected_keys`. + + Args: + expected_keys (`List[str]`, *optional*): + The list of the expected keys in the initialized model. + loaded_keys (`List[str]`, *optional*): + The list of the loaded keys in the checkpoint. + """ + return expected_keys + + def get_special_dtypes_update(self, model, torch_dtype: "torch.dtype") -> Dict[str, "torch.dtype"]: + """ + returns dtypes for modules that are not quantized - used for the computation of the device_map in case + one passes a str as a device_map. The method will use the `modules_to_not_convert` that is modified + in `_process_model_before_weight_loading`. + + Args: + model (`~transformers.PreTrainedModel`): + The model to quantize + torch_dtype (`torch.dtype`): + The dtype passed in `from_pretrained` method. + """ + + return { + name: torch_dtype + for name, _ in model.named_parameters() + if any(m in name for m in self.modules_to_not_convert) + } + + def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + """adjust max_memory argument for infer_auto_device_map() if extra memory is needed for quantization""" + return max_memory + + def check_quantized_param( + self, + model: "PreTrainedModel", + param_value: "torch.Tensor", + param_name: str, + state_dict: Dict[str, Any], + **kwargs, + ) -> bool: + """ + checks if a loaded state_dict component is part of quantized param + some validation; only defined if + requires_parameters_quantization == True for quantization methods that require to create a new parameters + for quantization. + """ + return False + + def create_quantized_param(self, *args, **kwargs) -> "torch.nn.Parameter": + """ + takes needed components from state_dict and creates quantized param; only applicable if + requires_parameters_quantization == True + """ + if not self.requires_parameters_quantization: + raise AttributeError( + f"`.create_quantized_param()` method is not supported by quantizer class {self.__class__.__name__}." + ) + + def validate_environment(self, *args, **kwargs): + """ + This method is used to potentially check for potential conflicts with arguments that are + passed in `from_pretrained`. You need to define it for all future quantizers that are integrated with transformers. + If no explicit check are needed, simply return nothing. + """ + return + + def update_tp_plan(self, config): + "updates the tp plan for the scales" + return config + + def preprocess_model(self, model: "PreTrainedModel", **kwargs): + """ + Setting model attributes and/or converting model before weights loading. At this point + the model should be initialized on the meta device so you can freely manipulate the skeleton + of the model in order to replace modules in-place. Make sure to override the abstract method `_process_model_before_weight_loading`. + + Args: + model (`~transformers.PreTrainedModel`): + The model to quantize + kwargs (`dict`, *optional*): + The keyword arguments that are passed along `_process_model_before_weight_loading`. + """ + model.is_quantized = True + model.quantization_method = self.quantization_config.quant_method + if self.pre_quantized: + self._convert_model_for_quantization(model) + return self._process_model_before_weight_loading(model, **kwargs) + + def postprocess_model(self, model: "PreTrainedModel", **kwargs): + """ + Post-process the model post weights loading. + Make sure to override the abstract method `_process_model_after_weight_loading`. + + Args: + model (`~transformers.PreTrainedModel`): + The model to quantize + kwargs (`dict`, *optional*): + The keyword arguments that are passed along `_process_model_after_weight_loading`. + """ + return self._process_model_after_weight_loading(model, **kwargs) + + def dequantize(self, model): + """ + Potentially dequantize the model to retrive the original model, with some loss in accuracy / performance. + Note not all quantization schemes support this. + """ + model = self._dequantize(model) + + # Delete quantizer and quantization config + del model.hf_quantizer + del model.config.quantization_config + del model.config._pre_quantization_dtype + model.is_quantized = False + + return model + + def _dequantize(self, model): + raise NotImplementedError( + f"{self.quantization_config.quant_method} has no implementation of `dequantize`, please raise an issue on GitHub." + ) + + @staticmethod + def get_modules_to_not_convert( + model: "PreTrainedModel", + skip_modules: Optional[List[str]] = None, + keep_in_fp32_modules: Optional[List[str]] = None, + ): + from ..integrations import get_keys_to_not_convert + + modules_to_not_convert = [] + if skip_modules is None: + modules_to_not_convert = get_keys_to_not_convert(model) + else: + modules_to_not_convert = skip_modules + + if keep_in_fp32_modules is not None: + modules_to_not_convert.extend(keep_in_fp32_modules) + + return modules_to_not_convert + + @property + def is_qat_trainable(self) -> bool: + """Flag indicating whether the quantized model can carry out quantization aware training""" + return False + + @property + def is_compileable(self) -> bool: + """Flag indicating whether the quantized model can be compiled""" + return False + + @abstractmethod + def _process_model_before_weight_loading(self, model, **kwargs): ... + + @abstractmethod + def _process_model_after_weight_loading(self, model, **kwargs): ... + + @abstractmethod + def is_serializable(self, safe_serialization=None): ... + + @property + @abstractmethod + def is_trainable(self): ... + + def _convert_model_for_quantization(self, model): + from accelerate import init_empty_weights + + for name, module in model.named_modules(): + module_class_name = module.__class__.__name__ + if ( + module_class_name in MODULES_TO_PATCH_FOR_QUANTIZATION.keys() + and self.quantization_config.quant_method == QuantizationMethod.COMPRESSED_TENSORS + ): + with init_empty_weights(): + parent_module, name = get_module_from_name(model, name) + parent_module._modules[name] = MODULES_TO_PATCH_FOR_QUANTIZATION[module_class_name]( + model.config.get_text_config() + ) + + +class SequentialLlama4TextExperts(ModuleList): + """ + A module that implements a compressed version of a list of expert modules. + This is specifically designed to work with Llama4TextExperts in MoE layers. + """ + + def __init__(self, config): + from transformers.models.llama4.modeling_llama4 import Llama4TextMLP + + super().__init__([Llama4TextMLP(config) for _ in range(config.num_local_experts)]) + self.num_experts = config.num_local_experts + + def forward( + self, + hidden_states: "torch.Tensor", + ) -> "torch.Tensor": + hidden_states = hidden_states.reshape(self.num_experts, -1, hidden_states.shape[-1]) + routed_out = torch.zeros_like(hidden_states) + for expert_idx in range(self.num_experts): + routed_out[expert_idx] = self[expert_idx](hidden_states[expert_idx]) + return routed_out + + +MODULES_TO_PATCH_FOR_QUANTIZATION = { + "Llama4TextExperts": { + "module_name": SequentialLlama4TextExperts, + "quantization_methods": [ + QuantizationMethod.COMPRESSED_TENSORS, + QuantizationMethod.BITS_AND_BYTES, + ], + } +} diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/beam_constraints.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/beam_constraints.py new file mode 100644 index 0000000000000000000000000000000000000000..daf64209b796772ac5c8da0dc139d944c2e82430 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/beam_constraints.py @@ -0,0 +1,524 @@ +from abc import ABC, abstractmethod +from typing import List, Optional + + +class Constraint(ABC): + r"""Abstract base class for all constraints that can be applied during generation. + It must define how the constraint can be satisfied. + + All classes that inherit Constraint must follow the requirement that + + ```py + completed = False + while not completed: + _, completed = constraint.update(constraint.advance()) + ``` + + will always terminate (halt). + """ + + def __init__(self): + # test for the above condition + self.test() + + def test(self): + """ + Tests whether this constraint has been properly defined. + """ + counter = 0 + completed = False + while not completed: + if counter == 1: + self.reset() + advance = self.advance() + if not self.does_advance(advance): + raise Exception( + "Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true." + ) + + stepped, completed, reset = self.update(advance) + counter += 1 + + if counter > 10000: + raise Exception("update() does not fulfill the constraint.") + + if self.remaining() != 0: + raise Exception("Custom Constraint is not defined correctly.") + + @abstractmethod + def advance(self): + """ + When called, returns the token(s) that would take this constraint one step closer to being fulfilled. + + Return: + token_ids (Union[int, List[int], None]): + - A single token ID (int) that advances the constraint, or + - A list of token IDs that could advance the constraint + - None if the constraint is completed or cannot be advanced + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + @abstractmethod + def does_advance(self, token_id: int): + """ + Reads in a token and returns whether it creates progress. + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + @abstractmethod + def update(self, token_id: int): + """ + Reads in a token and returns booleans that indicate the progress made by it. This function will update the + state of this object unlikes `does_advance(self, token_id: int)`. + + This isn't to test whether a certain token will advance the progress; it's to update its state as if it has + been generated. This becomes important if token_id != desired token (refer to else statement in + PhrasalConstraint) + + Args: + token_id(`int`): + The id of a newly generated token in the beam search. + Return: + stepped(`bool`): + Whether this constraint has become one step closer to being fulfuilled. + completed(`bool`): + Whether this constraint has been completely fulfilled by this token being generated. + reset (`bool`): + Whether this constraint has reset its progress by this token being generated. + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + @abstractmethod + def reset(self): + """ + Resets the state of this constraint to its initialization. We would call this in cases where the fulfillment of + a constraint is abrupted by an unwanted token. + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + @abstractmethod + def remaining(self): + """ + Returns the number of remaining steps of `advance()` in order to complete this constraint. + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + @abstractmethod + def copy(self, stateful=False): + """ + Creates a new instance of this constraint. + + Args: + stateful(`bool`): Whether to not only copy the constraint for new instance, but also its state. + + Return: + constraint(`Constraint`): The same constraint as the one being called from. + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + +class PhrasalConstraint(Constraint): + r""" + [`Constraint`] enforcing that an ordered sequence of tokens is included in the output. + + Args: + token_ids (`List[int]`): + The id of the token that must be generated by the output. + """ + + def __init__(self, token_ids: List[int]): + super(Constraint, self).__init__() + + if not isinstance(token_ids, list) or len(token_ids) == 0: + raise ValueError(f"`token_ids` has to be a non-empty list, but is {token_ids}.") + if any((not isinstance(token_id, int) or token_id < 0) for token_id in token_ids): + raise ValueError(f"Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.") + + self.token_ids = token_ids + + self.seqlen = len(self.token_ids) + self.fulfilled_idx = -1 # the index of the currently fulfilled step + self.completed = False + + def advance(self): + if self.completed: + return None + return self.token_ids[self.fulfilled_idx + 1] + + def does_advance(self, token_id: int): + if not isinstance(token_id, int): + raise TypeError(f"`token_id` has to be an `int`, but is {token_id} of type {type(token_id)}") + + if self.completed: + return False + + return token_id == self.token_ids[self.fulfilled_idx + 1] + + def update(self, token_id: int): + if not isinstance(token_id, int): + raise TypeError(f"`token_id` has to be an `int`, but is {token_id} of type {type(token_id)}") + + stepped = False + completed = False + reset = False + + if self.does_advance(token_id): + self.fulfilled_idx += 1 + stepped = True + if self.fulfilled_idx == (self.seqlen - 1): + completed = True + self.completed = completed + else: + # failed to make progress. + reset = True + self.reset() + return stepped, completed, reset + + def reset(self): + self.completed = False + self.fulfilled_idx = 0 + + def remaining(self): + return self.seqlen - (self.fulfilled_idx + 1) + + def copy(self, stateful=False): + new_constraint = PhrasalConstraint(self.token_ids) + + if stateful: + new_constraint.seq_len = self.seqlen + new_constraint.fulfilled_idx = self.fulfilled_idx + new_constraint.completed = self.completed + + return new_constraint + + +class DisjunctiveTrie: + def __init__(self, nested_token_ids: List[List[int]], no_subsets=True): + r""" + A helper class that builds a trie with the words represented in `nested_token_ids`. + """ + self.max_height = max([len(one) for one in nested_token_ids]) + + root = {} + for token_ids in nested_token_ids: + level = root + for tidx, token_id in enumerate(token_ids): + if token_id not in level: + level[token_id] = {} + + level = level[token_id] + + if no_subsets and self.has_subsets(root, nested_token_ids): + raise ValueError( + "Each list in `nested_token_ids` can't be a complete subset of another list, but is" + f" {nested_token_ids}." + ) + + self.trie = root + + def next_tokens(self, current_seq): + """ + The next possible tokens that will progress the trie, given the current sequence of tokens in `current_seq`. + """ + start = self.trie + + for current_token in current_seq: + start = start[current_token] + + next_tokens = list(start.keys()) + + return next_tokens + + def reached_leaf(self, current_seq): + next_tokens = self.next_tokens(current_seq) + + return len(next_tokens) == 0 + + def count_leaves(self, root): + next_nodes = list(root.values()) + if len(next_nodes) == 0: + return 1 + else: + return sum([self.count_leaves(nn) for nn in next_nodes]) + + def has_subsets(self, trie, nested_token_ids): + """ + Returns whether # of leaves == # of words. Otherwise some word is a subset of another. + """ + leaf_count = self.count_leaves(trie) + return len(nested_token_ids) != leaf_count + + +class DisjunctiveConstraint(Constraint): + r""" + A special [`Constraint`] that is fulfilled by fulfilling just one of several constraints. + + Args: + nested_token_ids (`List[List[int]]`): + A list of words, where each word is a list of ids. This constraint is fulfilled by generating just one from + the list of words. + """ + + def __init__(self, nested_token_ids: List[List[int]]): + super(Constraint, self).__init__() + + if not isinstance(nested_token_ids, list) or len(nested_token_ids) == 0: + raise ValueError(f"`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.") + if any(not isinstance(token_ids, list) for token_ids in nested_token_ids): + raise ValueError(f"`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.") + if any( + any((not isinstance(token_id, int) or token_id < 0) for token_id in token_ids) + for token_ids in nested_token_ids + ): + raise ValueError( + f"Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}." + ) + + self.trie = DisjunctiveTrie(nested_token_ids) + self.token_ids = nested_token_ids + + self.seqlen = self.trie.max_height + self.current_seq = [] + self.completed = False + + def advance(self): + token_list = self.trie.next_tokens(self.current_seq) + + if len(token_list) == 0: + return None + else: + return token_list + + def does_advance(self, token_id: int): + if not isinstance(token_id, int): + raise TypeError(f"`token_id` is supposed to be type `int`, but is {token_id} of type {type(token_id)}") + + next_tokens = self.trie.next_tokens(self.current_seq) + + return token_id in next_tokens + + def update(self, token_id: int): + if not isinstance(token_id, int): + raise TypeError(f"`token_id` is supposed to be type `int`, but is {token_id} of type {type(token_id)}") + + stepped = False + completed = False + reset = False + + if self.does_advance(token_id): + self.current_seq.append(token_id) + stepped = True + else: + reset = True + self.reset() + + completed = self.trie.reached_leaf(self.current_seq) + self.completed = completed + + return stepped, completed, reset + + def reset(self): + self.completed = False + self.current_seq = [] + + def remaining(self): + if self.completed: + # since this can be completed without reaching max height + return 0 + else: + return self.seqlen - len(self.current_seq) + + def copy(self, stateful=False): + new_constraint = DisjunctiveConstraint(self.token_ids) + + if stateful: + new_constraint.seq_len = self.seqlen + new_constraint.current_seq = self.current_seq + new_constraint.completed = self.completed + + return new_constraint + + +class ConstraintListState: + r""" + A class for beam scorers to track its progress through a list of constraints. + + Args: + constraints (`List[Constraint]`): + A list of [`Constraint`] objects that must be fulfilled by the beam scorer. + """ + + def __init__(self, constraints: List[Constraint]): + self.constraints = constraints + + # max # of steps required to fulfill a given constraint + self.max_seqlen = max([c.seqlen for c in constraints]) + self.n_constraints = len(constraints) + self.completed = False + + self.init_state() + + def init_state(self): + self.complete_constraints = [] + self.inprogress_constraint = None + self.pending_constraints = [constraint.copy(stateful=False) for constraint in self.constraints] + + def get_bank(self): + add = 0 + if self.inprogress_constraint: + # extra points for having a constraint mid-fulfilled + add += self.max_seqlen - self.inprogress_constraint.remaining() + + return (len(self.complete_constraints) * self.max_seqlen) + add + + def advance(self): + """The list of tokens to generate such that we can make progress. + By "list" we don't mean the list of token that will fully fulfill a constraint. + + Given constraints `c_i = {t_ij | j == # of tokens}`, If we're not in the middle of progressing through a + specific constraint `c_i`, we return: + + `[t_k1 for k in indices of unfulfilled constraints]` + + If we are in the middle of a constraint, then we return: + `[t_ij]`, where `i` is the index of the inprogress constraint, `j` is the next step for the constraint. + + Though we don't care which constraint is fulfilled first, if we are in the progress of fulfilling a constraint, + that's the only one we'll return. + """ + token_list = [] + if self.inprogress_constraint is None: + for constraint in self.pending_constraints: # "pending" == "unfulfilled yet" + advance = constraint.advance() + if isinstance(advance, int): + token_list.append(advance) + elif isinstance(advance, list): + token_list.extend(advance) + else: + advance = self.inprogress_constraint.advance() + if isinstance(advance, int): + token_list.append(advance) + elif isinstance(advance, list): + token_list.extend(advance) + + if len(token_list) == 0: + return None + else: + return token_list + + def reset(self, token_ids: Optional[List[int]]): + """ + token_ids: the tokens generated thus far to reset the state of the progress through constraints. + """ + self.init_state() + + if token_ids is not None: + for token in token_ids: + # completes or steps **one** constraint + complete, stepped = self.add(token) + + # the entire list of constraints are fulfilled + if self.completed: + break + + def add(self, token_id: int): + if not isinstance(token_id, int): + raise TypeError(f"`token_id` should be an `int`, but is `{token_id}`.") + + complete, stepped = False, False + + if self.completed: + complete = True + stepped = False + return complete, stepped + + if self.inprogress_constraint is not None: + # In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current + # job, simply update the state + + stepped, complete, reset = self.inprogress_constraint.update(token_id) + if reset: + # 1. If the next token breaks the progress, then we must restart. + # e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books". + + # But that doesn't mean we self.init_state(), since we only reset the state for this particular + # constraint, not the full list of constraints. + + self.pending_constraints.append(self.inprogress_constraint.copy(stateful=False)) + self.inprogress_constraint = None + + if complete: + # 2. If the next token completes the constraint, move it to completed list, set + # inprogress to None. If there are no pending constraints either, then this full list of constraints + # is complete. + + self.complete_constraints.append(self.inprogress_constraint) + self.inprogress_constraint = None + + if len(self.pending_constraints) == 0: + # we're done! + self.completed = True + + else: + # Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list + # of constraints? + + for cidx, pending_constraint in enumerate(self.pending_constraints): + if pending_constraint.does_advance(token_id): + stepped, complete, reset = pending_constraint.update(token_id) + + if not stepped: + raise Exception( + "`constraint.update(token_id)` is not yielding incremental progress, " + "even though `constraint.does_advance(token_id)` is true." + ) + + if complete: + self.complete_constraints.append(pending_constraint) + self.inprogress_constraint = None + + if not complete and stepped: + self.inprogress_constraint = pending_constraint + + if complete or stepped: + # If we made any progress at all, then it's at least not a "pending constraint". + + self.pending_constraints = ( + self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :] + ) + + if len(self.pending_constraints) == 0 and self.inprogress_constraint is None: + # If there's no longer any pending after this and no inprogress either, then we must be + # complete. + + self.completed = True + + break # prevent accidentally stepping through multiple constraints with just one token. + + return complete, stepped + + def copy(self, stateful=True): + new_state = ConstraintListState(self.constraints) # we actually never though self.constraints objects + # throughout this process. So it's at initialization state. + + if stateful: + new_state.complete_constraints = [ + constraint.copy(stateful=True) for constraint in self.complete_constraints + ] + if self.inprogress_constraint is not None: + new_state.inprogress_constraint = self.inprogress_constraint.copy(stateful=True) + new_state.pending_constraints = [constraint.copy() for constraint in self.pending_constraints] + + return new_state diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/beam_search.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/beam_search.py new file mode 100644 index 0000000000000000000000000000000000000000..3938deb4826025834967ec7782b47242c5d77899 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/beam_search.py @@ -0,0 +1,1013 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from collections import UserDict +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..utils import add_start_docstrings +from .beam_constraints import Constraint, ConstraintListState + + +PROCESS_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size * num_beams, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using any class inheriting from [`PreTrainedTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + next_scores (`torch.FloatTensor` of shape `(batch_size, 2 * num_beams)`): + Current scores of the top `2 * num_beams` non-finished beam hypotheses. + next_tokens (`torch.LongTensor` of shape `(batch_size, 2 * num_beams)`): + `input_ids` of the tokens corresponding to the top `2 * num_beams` non-finished beam hypotheses. + next_indices (`torch.LongTensor` of shape `(batch_size, 2 * num_beams)`): + Beam indices indicating to which beam hypothesis the `next_tokens` correspond. + pad_token_id (`int`, *optional*): + The id of the *padding* token. + eos_token_id (`Union[int, List[int]]`, *optional*): + The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens. + beam_indices (`torch.LongTensor`, *optional*): + Beam indices indicating to which beam hypothesis each token correspond. + group_index (`int`, *optional*): + The index of the group of beams. Used with [`~PreTrainedModel.group_beam_search`]. + + Return: + `UserDict`: A dictionary composed of the fields as defined above: + + - **next_beam_scores** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Updated scores of all + non-finished beams. + - **next_beam_tokens** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Next tokens to be added + to the non-finished beam_hypotheses. + - **next_beam_indices** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Beam indices + indicating to which beam the next tokens shall be added. + +""" + +FINALIZE_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size * num_beams, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using any class inheriting from [`PreTrainedTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + final_beam_scores (`torch.FloatTensor` of shape `(batch_size * num_beams)`): + The final scores of all non-finished beams. + final_beam_tokens (`torch.FloatTensor` of shape `(batch_size * num_beams)`): + The last tokens to be added to the non-finished beam_hypotheses. + final_beam_indices (`torch.FloatTensor` of shape `(batch_size * num_beams)`): + The beam indices indicating to which beam the `final_beam_tokens` shall be added. + pad_token_id (`int`, *optional*): + The id of the *padding* token. + eos_token_id (`Union[int, List[int]]`, *optional*): + The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens. + + Return: + `torch.LongTensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated sequences. + The second dimension (sequence_length) is either equal to `max_length` or shorter if all batches finished early + due to the `eos_token_id`. + +""" + + +class BeamScorer(ABC): + """ + Abstract base class for all beam scorers that are used for [`~PreTrainedModel.beam_search`] and + [`~PreTrainedModel.beam_sample`]. + """ + + @abstractmethod + @add_start_docstrings(PROCESS_INPUTS_DOCSTRING) + def process( + self, + input_ids: torch.LongTensor, + next_scores: torch.FloatTensor, + next_tokens: torch.LongTensor, + next_indices: torch.LongTensor, + **kwargs, + ) -> Tuple[torch.Tensor]: + raise NotImplementedError("This is an abstract method.") + + @abstractmethod + @add_start_docstrings(FINALIZE_INPUTS_DOCSTRING) + def finalize( + self, + input_ids: torch.LongTensor, + next_scores: torch.FloatTensor, + next_tokens: torch.LongTensor, + next_indices: torch.LongTensor, + max_length: int, + **kwargs, + ) -> torch.LongTensor: + raise NotImplementedError("This is an abstract method.") + + +class BeamSearchScorer(BeamScorer): + r""" + [`BeamScorer`] implementing standard beam search decoding. + + Adapted in part from [Facebook's XLM beam search + code](https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529). + + Reference for the diverse beam search algorithm and implementation [Ashwin Kalyan's DBS + implementation](https://github.com/ashwinkalyan/dbs/blob/master/dbs/beam_utils.lua) + + Args: + batch_size (`int`): + Batch Size of `input_ids` for which standard beam search decoding is run in parallel. + num_beams (`int`): + Number of beams for beam search. + device (`torch.device`): + Defines the device type (*e.g.*, `"cpu"` or `"cuda"`) on which this instance of `BeamSearchScorer` will be + allocated. + length_penalty (`float`, *optional*, defaults to 1.0): + Exponential penalty to the length that is used with beam-based generation. It is applied as an exponent to + the sequence length, which in turn is used to divide the score of the sequence. Since the score is the log + likelihood of the sequence (i.e. negative), `length_penalty` > 0.0 promotes longer sequences, while + `length_penalty` < 0.0 encourages shorter sequences. + do_early_stopping (`bool` or `str`, *optional*, defaults to `False`): + Controls the stopping condition for beam-based methods, like beam-search. It accepts the following values: + `True`, where the generation stops as soon as there are `num_beams` complete candidates; `False`, where an + heuristic is applied and the generation stops when is it very unlikely to find better candidates; + `"never"`, where the beam search procedure only stops when there cannot be better candidates (canonical + beam search algorithm). + num_beam_hyps_to_keep (`int`, *optional*, defaults to 1): + The number of beam hypotheses that shall be returned upon calling + [`~transformers.BeamSearchScorer.finalize`]. + num_beam_groups (`int`, *optional*, defaults to 1): + Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams. + See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details. + max_length (`int`, *optional*): + The maximum length of the sequence to be generated. + """ + + def __init__( + self, + batch_size: int, + num_beams: int, + device: torch.device, + length_penalty: Optional[float] = 1.0, + do_early_stopping: Optional[Union[bool, str]] = False, + num_beam_hyps_to_keep: Optional[int] = 1, + num_beam_groups: Optional[int] = 1, + max_length: Optional[int] = None, + ): + self.num_beams = num_beams + self.device = device + self.length_penalty = length_penalty + self.do_early_stopping = do_early_stopping + self.num_beam_hyps_to_keep = num_beam_hyps_to_keep + self.num_beam_groups = num_beam_groups + self.group_size = self.num_beams // self.num_beam_groups + + self._is_init = False + # self._beam_hyps[i*self.num_beam_groups+j] is the beam_hyps of the j-th group in the i-th mini-batch. + # If group_beam_search is not used, the list consists of `batch_size` beam_hyps. + self._beam_hyps = [ + BeamHypotheses( + num_beams=self.group_size, + length_penalty=self.length_penalty, + early_stopping=self.do_early_stopping, + max_length=max_length, + ) + for _ in range(batch_size * self.num_beam_groups) + ] + # self._done[i*self.num_beam_groups+j] indicates whether the generation of the beam_hyps of the j-th group + # in the i-th mini-batch is complete. + self._done = torch.tensor( + [False for _ in range(batch_size * self.num_beam_groups)], dtype=torch.bool, device=self.device + ) + + if not isinstance(num_beams, int) or num_beams <= 1: + raise ValueError( + f"`num_beams` has to be an integer strictly greater than 1, but is {num_beams}. For `num_beams` == 1," + " one should make use of `greedy_search` instead." + ) + + if not isinstance(num_beam_groups, int) or (num_beam_groups > num_beams) or (num_beams % num_beam_groups != 0): + raise ValueError( + "`num_beam_groups` has to be an integer smaller or equal than `num_beams` and `num_beams` has to be" + f" divisible by `num_beam_groups`, but is {num_beam_groups} with `num_beams` being {num_beams}." + ) + + @property + def is_done(self) -> bool: + return self._done.all() + + def process( + self, + input_ids: torch.LongTensor, + next_scores: torch.FloatTensor, + next_tokens: torch.LongTensor, + next_indices: torch.LongTensor, + pad_token_id: Optional[Union[int, torch.Tensor]] = None, + eos_token_id: Optional[Union[int, List[int], torch.Tensor]] = None, + beam_indices: Optional[torch.LongTensor] = None, + group_index: Optional[int] = 0, + decoder_prompt_len: Optional[int] = 0, + ) -> Dict[str, torch.Tensor]: + # add up to the length which the next_scores is calculated on (including decoder prompt) + cur_len = input_ids.shape[-1] + 1 + batch_size = len(self._beam_hyps) // self.num_beam_groups + + if not (batch_size == (input_ids.shape[0] // self.group_size)): + if self.num_beam_groups > 1: + raise ValueError( + f"A group beam size of {input_ids.shape[0]} is used as the input, but a group beam " + f"size of {self.group_size} is expected by the beam scorer." + ) + else: + raise ValueError( + f"A beam size of {input_ids.shape[0]} is used as the input, but a beam size of " + f"{self.group_size} is expected by the beam scorer." + ) + + device = input_ids.device + next_beam_scores = torch.zeros((batch_size, self.group_size), dtype=next_scores.dtype, device=device) + next_beam_tokens = torch.zeros((batch_size, self.group_size), dtype=next_tokens.dtype, device=device) + next_beam_indices = torch.zeros((batch_size, self.group_size), dtype=next_indices.dtype, device=device) + + if eos_token_id is not None and not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id) + + for batch_idx in range(batch_size): + batch_group_idx = batch_idx * self.num_beam_groups + group_index + if self._done[batch_group_idx]: + if self.num_beams < len(self._beam_hyps[batch_group_idx]): + raise ValueError(f"Batch can only be done if at least {self.num_beams} beams have been generated") + if eos_token_id is None or pad_token_id is None: + raise ValueError("Generated beams >= num_beams -> eos_token_id and pad_token have to be defined") + # pad the batch + next_beam_scores[batch_idx, :] = 0 + next_beam_tokens[batch_idx, :] = pad_token_id + next_beam_indices[batch_idx, :] = 0 + continue + + # next tokens for this sentence + beam_idx = 0 + for beam_token_rank, (next_token, next_score, next_index) in enumerate( + zip(next_tokens[batch_idx], next_scores[batch_idx], next_indices[batch_idx]) + ): + batch_beam_idx = batch_idx * self.group_size + next_index + # add to generated hypotheses if end of sentence + if (eos_token_id is not None) and (next_token.item() in eos_token_id): + # if beam_token does not belong to top num_beams tokens, it should not be added + is_beam_token_worse_than_top_num_beams = beam_token_rank >= self.group_size + if is_beam_token_worse_than_top_num_beams: + continue + if beam_indices is not None: + beam_index = beam_indices[batch_beam_idx] + beam_index = beam_index + (batch_beam_idx,) + else: + beam_index = None + + self._beam_hyps[batch_group_idx].add( + input_ids[batch_beam_idx].clone(), + next_score.item(), + beam_indices=beam_index, + generated_len=cur_len - decoder_prompt_len, + ) + else: + # add next predicted token since it is not eos_token + next_beam_scores[batch_idx, beam_idx] = next_score + next_beam_tokens[batch_idx, beam_idx] = next_token + next_beam_indices[batch_idx, beam_idx] = batch_beam_idx + beam_idx += 1 + + # once the beam for next step is full, don't add more tokens to it. + if beam_idx == self.group_size: + break + + if beam_idx < self.group_size: + raise ValueError( + f"At most {self.group_size} tokens in {next_tokens[batch_idx]} can be equal to `eos_token_id:" + f" {eos_token_id}`. Make sure {next_tokens[batch_idx]} are corrected." + ) + + # Check if we are done so that we can save a pad step if all(done) + self._done[batch_group_idx] = self._done[batch_group_idx] or self._beam_hyps[batch_group_idx].is_done( + next_scores[batch_idx].max().item(), cur_len, decoder_prompt_len + ) + + return UserDict( + { + "next_beam_scores": next_beam_scores.view(-1), + "next_beam_tokens": next_beam_tokens.view(-1), + "next_beam_indices": next_beam_indices.view(-1), + } + ) + + def finalize( + self, + input_ids: torch.LongTensor, + final_beam_scores: torch.FloatTensor, + final_beam_tokens: torch.LongTensor, + final_beam_indices: torch.LongTensor, + max_length: int, + pad_token_id: Optional[Union[int, torch.Tensor]] = None, + eos_token_id: Optional[Union[int, List[int], torch.Tensor]] = None, + beam_indices: Optional[torch.LongTensor] = None, + decoder_prompt_len: Optional[int] = 0, + ) -> Tuple[torch.LongTensor]: + batch_size = len(self._beam_hyps) // self.num_beam_groups + + if eos_token_id is not None and not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id) + + # finalize all open beam hypotheses and add to generated hypotheses + for batch_group_idx, beam_hyp in enumerate(self._beam_hyps): + if self._done[batch_group_idx]: + continue + + # all open beam hypotheses are added to the beam hypothesis + # beam hypothesis class automatically keeps the best beams + for index_per_group in range(self.group_size): + batch_beam_idx = batch_group_idx * self.group_size + index_per_group + final_score = final_beam_scores[batch_beam_idx].item() + final_tokens = input_ids[batch_beam_idx] + beam_index = beam_indices[batch_beam_idx] if beam_indices is not None else None + generated_len = final_tokens.shape[-1] - decoder_prompt_len + beam_hyp.add(final_tokens, final_score, beam_indices=beam_index, generated_len=generated_len) + + # select the best hypotheses + sent_lengths = input_ids.new(batch_size * self.num_beam_hyps_to_keep) + best = [] + best_indices = [] + best_scores = torch.zeros(batch_size * self.num_beam_hyps_to_keep, device=self.device, dtype=torch.float32) + + # retrieve best hypotheses + for i in range(batch_size): + beam_hyps_in_batch = self._beam_hyps[i * self.num_beam_groups : (i + 1) * self.num_beam_groups] + candidate_beams = [beam for beam_hyp in beam_hyps_in_batch for beam in beam_hyp.beams] + sorted_hyps = sorted(candidate_beams, key=lambda x: x[0]) + for j in range(self.num_beam_hyps_to_keep): + best_hyp_tuple = sorted_hyps.pop() + best_score = best_hyp_tuple[0] + best_hyp = best_hyp_tuple[1] + best_index = best_hyp_tuple[2] + sent_lengths[self.num_beam_hyps_to_keep * i + j] = len(best_hyp) + + # append hyp to lists + best.append(best_hyp) + + # append indices to list + best_indices.append(best_index) + + best_scores[i * self.num_beam_hyps_to_keep + j] = best_score + + # prepare for adding eos + sent_lengths_max = sent_lengths.max().item() + 1 + sent_max_len = min(sent_lengths_max, max_length) if max_length is not None else sent_lengths_max + decoded: torch.LongTensor = input_ids.new(batch_size * self.num_beam_hyps_to_keep, sent_max_len) + + if len(best_indices) > 0 and best_indices[0] is not None: + indices: torch.LongTensor = input_ids.new(batch_size * self.num_beam_hyps_to_keep, sent_max_len) + else: + indices = None + + # shorter batches are padded if needed + if sent_lengths.min().item() != sent_lengths.max().item(): + if pad_token_id is None: + raise ValueError("`pad_token_id` has to be defined") + decoded.fill_(pad_token_id) + + if indices is not None: + indices.fill_(-1) + + # fill with hypotheses and eos_token_id if the latter fits in + for i, (hypo, best_idx) in enumerate(zip(best, best_indices)): + decoded[i, : sent_lengths[i]] = hypo + + if indices is not None: + indices[i, : len(best_idx)] = torch.tensor(best_idx) + + if sent_lengths[i] < sent_max_len: + # inserting only the first eos_token_id + decoded[i, sent_lengths[i]] = eos_token_id[0] + + return UserDict( + { + "sequences": decoded, + "sequence_scores": best_scores, + "beam_indices": indices, + } + ) + + +class ConstrainedBeamSearchScorer(BeamScorer): + r""" + [`BeamScorer`] implementing constrained beam search decoding. + + + Args: + batch_size (`int`): + Batch Size of `input_ids` for which standard beam search decoding is run in parallel. + num_beams (`int`): + Number of beams for beam search. + constraints (`List[Constraint]`): + A list of positive constraints represented as `Constraint` objects that must be fulfilled in the generation + output. For more information, the documentation of [`Constraint`] should be read. + device (`torch.device`): + Defines the device type (*e.g.*, `"cpu"` or `"cuda"`) on which this instance of `BeamSearchScorer` will be + allocated. + length_penalty (`float`, *optional*, defaults to 1.0): + Exponential penalty to the length that is used with beam-based generation. It is applied as an exponent to + the sequence length, which in turn is used to divide the score of the sequence. Since the score is the log + likelihood of the sequence (i.e. negative), `length_penalty` > 0.0 promotes longer sequences, while + `length_penalty` < 0.0 encourages shorter sequences. + do_early_stopping (`bool` or `str`, *optional*, defaults to `False`): + Controls the stopping condition for beam-based methods, like beam-search. It accepts the following values: + `True`, where the generation stops as soon as there are `num_beams` complete candidates; `False`, where an + heuristic is applied and the generation stops when is it very unlikely to find better candidates; + `"never"`, where the beam search procedure only stops when there cannot be better candidates (canonical + beam search algorithm). + num_beam_hyps_to_keep (`int`, *optional*, defaults to 1): + The number of beam hypotheses that shall be returned upon calling + [`~transformers.BeamSearchScorer.finalize`]. + num_beam_groups (`int`, *optional*, defaults to 1): + Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams. + See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details. + max_length (`int`, *optional*): + The maximum length of the sequence to be generated. + """ + + def __init__( + self, + batch_size: int, + num_beams: int, + constraints: List[Constraint], + device: torch.device, + length_penalty: Optional[float] = 1.0, + do_early_stopping: Optional[Union[bool, str]] = False, + num_beam_hyps_to_keep: Optional[int] = 1, + num_beam_groups: Optional[int] = 1, + max_length: Optional[int] = None, + ): + self.num_beams = num_beams + self.device = device + self.length_penalty = length_penalty + self.do_early_stopping = do_early_stopping + self.num_beam_hyps_to_keep = num_beam_hyps_to_keep + self.num_beam_groups = num_beam_groups + self.group_size = self.num_beams // self.num_beam_groups + self.constraints = constraints + + self._is_init = False + self._beam_hyps = [ + BeamHypotheses( + num_beams=self.num_beams, + length_penalty=self.length_penalty, + early_stopping=self.do_early_stopping, + max_length=max_length, + ) + for _ in range(batch_size) + ] + self._done = torch.tensor([False for _ in range(batch_size)], dtype=torch.bool, device=self.device) + + if not isinstance(num_beams, int) or num_beams <= 1: + raise ValueError( + f"`num_beams` has to be an integer strictly greater than 1, but is {num_beams}. For `num_beams` == 1," + " one should make use of `greedy_search` instead." + ) + + if not isinstance(num_beam_groups, int) or (num_beam_groups > num_beams) or (num_beams % num_beam_groups != 0): + raise ValueError( + "`num_beam_groups` has to be an integer smaller or equal than `num_beams` and `num_beams` has to be" + f" divisible by `num_beam_groups`, but is {num_beam_groups} with `num_beams` being {num_beams}." + ) + + @property + def is_done(self) -> bool: + return self._done.all() + + def make_constraint_states(self, n): + return [ConstraintListState([constraint.copy() for constraint in self.constraints]) for _ in range(n)] + + def check_completes_constraints(self, sequence): + new_state = self.make_constraint_states(1)[0] + new_state.reset(sequence) + return new_state.completed + + def process( + self, + input_ids: torch.LongTensor, + next_scores: torch.FloatTensor, + next_tokens: torch.LongTensor, + next_indices: torch.LongTensor, + scores_for_all_vocab: torch.FloatTensor, + pad_token_id: Optional[Union[int, torch.Tensor]] = None, + eos_token_id: Optional[Union[int, List[int], torch.Tensor]] = None, + beam_indices: Optional[torch.LongTensor] = None, + decoder_prompt_len: Optional[int] = 0, + ) -> Tuple[torch.Tensor]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size * num_beams, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using any class inheriting from [`PreTrainedTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + next_scores (`torch.FloatTensor` of shape `(batch_size, 2 * num_beams)`): + Current scores of the top `2 * num_beams` non-finished beam hypotheses. + next_tokens (`torch.LongTensor` of shape `(batch_size, 2 * num_beams)`): + `input_ids` of the tokens corresponding to the top `2 * num_beams` non-finished beam hypotheses. + next_indices (`torch.LongTensor` of shape `(batch_size, 2 * num_beams)`): + Beam indices indicating to which beam hypothesis the `next_tokens` correspond. + scores_for_all_vocab (`torch.FloatTensor` of shape `(batch_size * num_beams, sequence_length)`): + The scores of all tokens in the vocabulary for each of the beam hypotheses. + pad_token_id (`int`, *optional*): + The id of the *padding* token. + eos_token_id (`Union[int, List[int]]`, *optional*): + The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens. + beam_indices (`torch.LongTensor`, *optional*): + Beam indices indicating to which beam hypothesis each token correspond. + decoder_prompt_len (`int`, *optional*): + The length of prompt that is included in the input to decoder. + Return: + `UserDict`: A dictionary composed of the fields as defined above: + + - **next_beam_scores** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Updated scores of + all + non-finished beams. + + - **next_beam_tokens** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Next tokens to be + added + to the non-finished beam_hypotheses. + - **next_beam_indices** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Beam indices + indicating to which beam the next tokens shall be added. + """ + + # add up to the length which the next_scores is calculated on (including decoder prompt) + cur_len = input_ids.shape[-1] + 1 + batch_size = len(self._beam_hyps) + if not (batch_size == (input_ids.shape[0] // self.group_size)): + if self.num_beam_groups > 1: + raise ValueError( + f"A group beam size of {input_ids.shape[0]} is used as the input, but a group beam " + f"size of {self.group_size} is expected by the beam scorer." + ) + else: + raise ValueError( + f"A beam size of {input_ids.shape[0]} is used as the input, but a beam size of " + f"{self.group_size} is expected by the beam scorer." + ) + + device = input_ids.device + + next_beam_scores = torch.zeros((batch_size, self.group_size), dtype=next_scores.dtype, device=device) + next_beam_tokens = torch.zeros((batch_size, self.group_size), dtype=next_tokens.dtype, device=device) + next_beam_indices = torch.zeros((batch_size, self.group_size), dtype=next_indices.dtype, device=device) + + if eos_token_id is not None and not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id) + + for batch_idx, beam_hyp in enumerate(self._beam_hyps): + if self._done[batch_idx]: + if self.num_beams < len(beam_hyp): + raise ValueError(f"Batch can only be done if at least {self.num_beams} beams have been generated") + if eos_token_id is None or pad_token_id is None: + raise ValueError("Generated beams >= num_beams -> eos_token_id and pad_token have to be defined") + # pad the batch + next_beam_scores[batch_idx, :] = 0 + next_beam_tokens[batch_idx, :] = pad_token_id + next_beam_indices[batch_idx, :] = 0 + continue + + # next tokens for this sentence. + beam_idx = 0 + for beam_token_rank, (next_token, next_score, next_index) in enumerate( + zip(next_tokens[batch_idx], next_scores[batch_idx], next_indices[batch_idx]) + ): + batch_beam_idx = batch_idx * self.group_size + next_index + # add to generated hypotheses if end of sentence + if (eos_token_id is not None) and (next_token.item() in eos_token_id): + # if beam_token does not belong to top num_beams tokens, it should not be added + is_beam_token_worse_than_top_num_beams = beam_token_rank >= self.group_size + if is_beam_token_worse_than_top_num_beams: + continue + + completes_constraint = self.check_completes_constraints(input_ids[batch_beam_idx].tolist()) + if completes_constraint: + if beam_indices is not None: + beam_index = beam_indices[batch_beam_idx] + beam_index = beam_index + (batch_beam_idx,) + else: + beam_index = None + + beam_hyp.add( + input_ids[batch_beam_idx].clone(), + next_score.item(), + beam_indices=beam_index, + generated_len=cur_len - decoder_prompt_len, + ) + else: + # add next predicted token since it is not eos_token + next_beam_scores[batch_idx, beam_idx] = next_score + next_beam_tokens[batch_idx, beam_idx] = next_token + next_beam_indices[batch_idx, beam_idx] = batch_beam_idx + beam_idx += 1 + + # once the beam for next step is full, don't add more tokens to it. + if beam_idx == self.group_size: + break + + new_scores, new_tokens, new_indices = self.step_sentence_constraint( + batch_idx, + input_ids, + scores_for_all_vocab, + next_beam_scores[batch_idx], + next_beam_tokens[batch_idx], + next_beam_indices[batch_idx], + ) + + next_beam_scores[batch_idx] = new_scores + next_beam_tokens[batch_idx] = new_tokens + next_beam_indices[batch_idx] = new_indices + + if beam_idx < self.group_size: + raise ValueError( + f"At most {self.group_size} tokens in {next_tokens[batch_idx]} can be equal to `eos_token_id:" + f" {eos_token_id}`. Make sure {next_tokens[batch_idx]} are corrected." + ) + + # Check if we are done so that we can save a pad step if all(done) + self._done[batch_idx] = self._done[batch_idx] or beam_hyp.is_done( + next_scores[batch_idx].max().item(), cur_len, decoder_prompt_len + ) + + return UserDict( + { + "next_beam_scores": next_beam_scores.view(-1), + "next_beam_tokens": next_beam_tokens.view(-1), + "next_beam_indices": next_beam_indices.view(-1), + } + ) + + def step_sentence_constraint( + self, + batch_idx: int, + input_ids: torch.LongTensor, + vocab_scores: torch.FloatTensor, + sent_beam_scores: torch.FloatTensor, + sent_beam_tokens: torch.LongTensor, + sent_beam_indices: torch.LongTensor, + push_progress: bool = False, + ): + # sent_beam_tokens are the next {num_beams} number of tokens that are under consideration for this beam + # (candidate next tokens) + + # 1. Adding "advance_tokens" + # using ConstraintStateList.advance(), we propose new tokens to be added into this "candidate list" that will + # advance us in fulfilling the constraints. + + # 2. Selecting best candidates such that we end up with highest probable candidates + # that fulfill our constraints. + + orig_len = sent_beam_indices.size(0) + device = sent_beam_indices.device + + # initialize states + topk_contraint_states = self.make_constraint_states(orig_len) + advance_constraint_states = self.make_constraint_states(orig_len) + + sidx, eidx = batch_idx * orig_len, (batch_idx + 1) * orig_len + this_batch_input_ids = input_ids[sidx:eidx] + this_batch_token_scores = vocab_scores[sidx:eidx] + full_hypotheses = torch.cat((input_ids[sent_beam_indices], sent_beam_tokens.unsqueeze(-1)), dim=-1) + + # need to make new hypothesis that advance the constraints + track_new = { + "new_seqs": full_hypotheses.tolist(), + "new_states": [], + "new_indices": [], + "new_tokens": [], + "new_scores": [], + } + for seq_idx, pre_seq in enumerate(this_batch_input_ids): + # pre_seq = ith sequence generated before this step. + + # input_ids -> (topk) generic beam search best model next tokens + # -> (advance) constraints forcing the next token + # either way, we need to sort them into "banks" later, so store a "ConstraintListState" for all types of + # hypotheses. + + topk_state = topk_contraint_states[seq_idx] + topk_state.reset(full_hypotheses[seq_idx].tolist()) + + advance_state = advance_constraint_states[seq_idx] + advance_state.reset(pre_seq.tolist()) + + if not advance_state.completed: + advance_tokens = torch.tensor(advance_state.advance(), dtype=torch.long, device=device) + for advance_token in advance_tokens: + # since adding each `advance_token` leads to a different hypothesis, create new state instance. + new_state = advance_state.copy(stateful=True) + new_state.add(advance_token.tolist()) + + advance_seq = torch.cat((pre_seq, advance_token.unsqueeze(0)), -1).tolist() + if advance_seq not in track_new["new_seqs"]: + # prevent duplicates, which are basically bound to happen in this process. + track_new["new_seqs"].append(advance_seq) + track_new["new_indices"].append(sidx + seq_idx) # idx -> global idx across all the batches + track_new["new_tokens"].append(advance_token) + track_new["new_scores"].append(this_batch_token_scores[seq_idx].take(advance_token)) + track_new["new_states"].append(new_state) + elif push_progress: + # Basically, `sent_beam_indices` often chooses very little among `input_ids` the generated sequences that + # actually fulfill our constraints. For example, let constraints == ["loves pies"] and + + # pre_seq_1 = "The child loves pies and" pre_seq_2 = "The child plays in the playground and" + + # Without this step, if `sent_beam_indices` is something like [1,1], then + # 1. `pre_seq_1` won't be added to the list of (topk) hypothesis since it's not in the indices and + # 2. it won't be added to the list of (advance) hypothesis since it's completed already. (this is + # the else part of `if constraints_completed[seq_idx]`) + # 3. it ends up simply getting removed from consideration. + + # #3 might be fine and actually desired, since it's likely that it's a low-probability output anyways, + # especially if it's not in the list of `sent_beam_indices`. But this often leads to lengthened beam + # search times, since completed sequences keep getting removed after all this effort for constrained + # generation. + + # Here, we basically take `pre_seq_1` and to "push" it into the considered list of hypotheses, by simply + # appending the next likely token in the vocabulary and adding it to the list of hypotheses. + + new_score, new_token = torch.max(this_batch_token_scores[seq_idx], 0) # some next probable token + advance_seq = torch.cat((pre_seq, new_token.unsqueeze(0)), -1) + + advance_state = advance_constraint_states[seq_idx] + + advance_seq = advance_seq.tolist() + + advance_state.reset(advance_seq) + if advance_seq not in track_new["new_seqs"]: + # but still don't want to have duplicates + track_new["new_seqs"].append(advance_seq) + track_new["new_indices"].append(seq_idx) + track_new["new_tokens"].append(new_token) + track_new["new_scores"].append(new_score) + track_new["new_states"].append(advance_state) + + if len(track_new["new_indices"]) > 0: + new_indices = torch.tensor(track_new["new_indices"], device=device) + new_tokens = torch.stack(track_new["new_tokens"]).to(device) + new_scores = torch.stack(track_new["new_scores"]).to(device) + + all_states = topk_contraint_states + track_new["new_states"] + all_tokens = torch.cat((sent_beam_tokens, new_tokens), -1) + all_scores = torch.cat((sent_beam_scores, new_scores), -1) + all_banks = torch.tensor([one.get_bank() for one in all_states], device=device) + + zipped = all_banks * 100 + all_scores + indices = zipped.sort(descending=True).indices + sorted_banks = all_banks[indices] + + # Then we end up with {sorted among bank C}, {sorted among bank C-1}, ..., {sorted among bank 0} + + counter = -1 + cur_bank = sorted_banks[0] + increments = [] + for bank in sorted_banks: + if bank == cur_bank: + counter += 1 + else: + counter = 0 + cur_bank = bank + increments.append(counter) + rearrangers = torch.tensor(np.argsort(increments, kind="mergesort")) + + indices = indices[rearrangers][:orig_len] + + sent_beam_scores = all_scores[indices] + sent_beam_tokens = all_tokens[indices] + sent_beam_indices = torch.cat((sent_beam_indices, new_indices))[indices] + + return sent_beam_scores, sent_beam_tokens, sent_beam_indices + + def finalize( + self, + input_ids: torch.LongTensor, + final_beam_scores: torch.FloatTensor, + final_beam_tokens: torch.LongTensor, + final_beam_indices: torch.LongTensor, + max_length: int, + pad_token_id: Optional[Union[int, torch.Tensor]] = None, + eos_token_id: Optional[Union[int, List[int], torch.Tensor]] = None, + beam_indices: Optional[torch.LongTensor] = None, + decoder_prompt_len: Optional[int] = 0, + ) -> Tuple[torch.LongTensor]: + batch_size = len(self._beam_hyps) + + if eos_token_id is not None and not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id) + + # finalize all open beam hypotheses and add to generated hypotheses + for batch_idx, beam_hyp in enumerate(self._beam_hyps): + if self._done[batch_idx]: + continue + + # all open beam hypotheses are added to the beam hypothesis + # beam hypothesis class automatically keeps the best beams + + ids_collect = [] + for beam_id in range(self.num_beams): + batch_beam_idx = batch_idx * self.num_beams + beam_id + final_score = final_beam_scores[batch_beam_idx].item() + final_tokens = input_ids[batch_beam_idx] + + completes_constraint = self.check_completes_constraints(final_tokens.tolist()) + if completes_constraint: + beam_index = beam_indices[batch_beam_idx] if beam_indices is not None else None + generated_len = final_tokens.shape[-1] - decoder_prompt_len + beam_hyp.add(final_tokens, final_score, beam_indices=beam_index, generated_len=generated_len) + ids_collect.append(beam_id) + + # due to overly complex constraints or other factors, sometimes we can't gaurantee a successful + # generation. In these cases we simply return the highest scoring outputs. + if len(ids_collect) < self.num_beam_hyps_to_keep: + for beam_id in range(self.num_beams): + if beam_id not in ids_collect: + batch_beam_idx = batch_idx * self.num_beams + beam_id + final_score = final_beam_scores[batch_beam_idx].item() + final_tokens = input_ids[batch_beam_idx] + generated_len = final_tokens.shape[-1] - decoder_prompt_len + beam_hyp.add(final_tokens, final_score, generated_len=generated_len) + if len(ids_collect) >= self.num_beam_hyps_to_keep: + break + + # select the best hypotheses + sent_lengths = input_ids.new(batch_size * self.num_beam_hyps_to_keep) + best = [] + best_indices = [] + best_scores = torch.zeros(batch_size * self.num_beam_hyps_to_keep, device=self.device, dtype=torch.float32) + + # retrieve best hypotheses + for i, beam_hyp in enumerate(self._beam_hyps): + sorted_hyps = sorted(beam_hyp.beams, key=lambda x: x[0]) + for j in range(self.num_beam_hyps_to_keep): + best_hyp_tuple = sorted_hyps.pop() + best_score = best_hyp_tuple[0] + best_hyp = best_hyp_tuple[1] + best_index = best_hyp_tuple[2] + sent_lengths[self.num_beam_hyps_to_keep * i + j] = len(best_hyp) + + # append to lists + best.append(best_hyp) + + # append indices to list + best_indices.append(best_index) + + best_scores[i * self.num_beam_hyps_to_keep + j] = best_score + + # prepare for adding eos + sent_lengths_max = sent_lengths.max().item() + 1 + + sent_max_len = min(sent_lengths_max, max_length) if max_length is not None else sent_lengths_max + decoded: torch.LongTensor = input_ids.new(batch_size * self.num_beam_hyps_to_keep, sent_max_len) + + if len(best_indices) > 0 and best_indices[0] is not None: + indices: torch.LongTensor = input_ids.new(batch_size * self.num_beam_hyps_to_keep, sent_max_len) + else: + indices = None + + # shorter batches are padded if needed + if sent_lengths.min().item() != sent_lengths.max().item(): + if pad_token_id is None: + raise ValueError("`pad_token_id` has to be defined") + decoded.fill_(pad_token_id) + + if indices is not None: + indices.fill_(-1) + + # fill with hypotheses and eos_token_id if the latter fits in + for i, (hypo, best_idx) in enumerate(zip(best, best_indices)): + decoded[i, : sent_lengths[i]] = hypo + + if indices is not None: + indices[i, : len(best_idx)] = torch.tensor(best_idx) + + if sent_lengths[i] < sent_max_len: + # inserting only the first eos_token_id + decoded[i, sent_lengths[i]] = eos_token_id[0] + + return UserDict( + { + "sequences": decoded, + "sequence_scores": best_scores, + "beam_indices": indices, + } + ) + + +class BeamHypotheses: + def __init__(self, num_beams: int, length_penalty: float, early_stopping: bool, max_length: Optional[int] = None): + """ + Initialize n-best list of hypotheses. + """ + self.length_penalty = length_penalty + self.early_stopping = early_stopping + self.max_length = max_length + self.num_beams = num_beams + self.beams = [] + self.worst_score = 1e9 + + if not isinstance(self.early_stopping, bool) and self.max_length is None: + raise ValueError( + "When `do_early_stopping` is set to a string, `max_length` must be defined. Ensure it is passed to the" + " BeamScorer class instance at initialization time." + ) + + def __len__(self): + """ + Number of hypotheses in the list. + """ + return len(self.beams) + + def add( + self, + hyp: torch.LongTensor, + sum_logprobs: float, + beam_indices: Optional[torch.LongTensor] = None, + generated_len: Optional[int] = None, + ): + """ + Add a new hypothesis to the list. + """ + if generated_len is not None: + score = sum_logprobs / (generated_len**self.length_penalty) + # This 'else' case exists for retrocompatibility + else: + score = sum_logprobs / (hyp.shape[-1] ** self.length_penalty) + + if len(self) < self.num_beams or score > self.worst_score: + self.beams.append((score, hyp, beam_indices)) + if len(self) > self.num_beams: + sorted_next_scores = sorted([(s, idx) for idx, (s, _, _) in enumerate(self.beams)]) + del self.beams[sorted_next_scores[0][1]] + self.worst_score = sorted_next_scores[1][0] + else: + self.worst_score = min(score, self.worst_score) + + def is_done(self, best_sum_logprobs: float, cur_len: int, decoder_prompt_len: Optional[int] = 0) -> bool: + """ + If there are enough hypotheses and that none of the hypotheses being generated can become better than the worst + one in the heap, then we are done with this sentence. + """ + + if len(self) < self.num_beams: + return False + + # `True`: stop as soon as at least `num_beams` hypotheses are finished + if self.early_stopping is True: + return True + # `False`: heuristic -- compute best possible score from `cur_len`, even though it is not entirely accurate + # when `length_penalty` is positive. See the discussion below for more details. + # https://github.com/huggingface/transformers/pull/20901#issuecomment-1369845565 + elif self.early_stopping is False: + highest_attainable_score = best_sum_logprobs / (cur_len - decoder_prompt_len) ** self.length_penalty + ret = self.worst_score >= highest_attainable_score + return ret + # `"never"`: compute the best possible score, depending on the signal of `length_penalty` + else: + # `length_penalty` > 0.0 -> max denominator is obtaned from `max_length`, not from `cur_len` -> min + # abs(`highest_attainable_score`) is obtained -> `highest_attainable_score` is negative, hence we obtain + # its max this way + if self.length_penalty > 0.0: + if self.max_length <= decoder_prompt_len: + raise ValueError("max_length is not larger than decoder prompt length") + highest_attainable_score = ( + best_sum_logprobs / (self.max_length - decoder_prompt_len) ** self.length_penalty + ) + # the opposite logic applies here (max `highest_attainable_score` from `cur_len`) + else: + highest_attainable_score = best_sum_logprobs / (cur_len - decoder_prompt_len) ** self.length_penalty + ret = self.worst_score >= highest_attainable_score + return ret diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/bitnet.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/bitnet.py new file mode 100644 index 0000000000000000000000000000000000000000..0b50f9738afb69e680371b5f868bc4508f2a60e7 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/bitnet.py @@ -0,0 +1,288 @@ +from ..utils import is_accelerate_available, is_torch_available, logging + + +if is_accelerate_available(): + from accelerate import init_empty_weights + +if is_torch_available(): + import torch + import torch.nn as nn + import torch.nn.functional as F + +logger = logging.get_logger(__name__) + + +# the weights are ternary so can be represented with 2 bits, and they are packed in uint8 tensors, hence the number of values per item is 4 +VALUES_PER_ITEM = 4 + + +def pack_weights(quantized_weights: torch.Tensor) -> torch.Tensor: + """ + Packs a tensor of quantized weights into a compact format using 2 bits per value. + + Parameters: + ----------- + quantized_weights : torch.Tensor + A tensor containing ternary quantized weights with values in {-1, 0, 1}. These values are adjusted to + {0, 1, 2} before being packed. + + Returns: + -------- + torch.Tensor + A packed tensor where each element stores 4 quantized values (each using 2 bits) in an 8-bit format. + """ + + original_shape = quantized_weights.shape + + row_dim = (original_shape[0] + VALUES_PER_ITEM - 1) // VALUES_PER_ITEM + + if len(original_shape) == 1: + packed_tensor_shape = (row_dim,) + else: + packed_tensor_shape = (row_dim, *original_shape[1:]) + + quantized_weights += 1 + packed = torch.zeros(packed_tensor_shape, device=quantized_weights.device, dtype=torch.uint8) + unpacked = quantized_weights.to(torch.uint8) + + it = min(VALUES_PER_ITEM, (original_shape[0] // row_dim) + 1) + for i in range(it): + start = i * row_dim + end = min(start + row_dim, original_shape[0]) + packed[: (end - start)] |= unpacked[start:end] << 2 * i + + return packed + + +@torch.compile +def unpack_weights(packed: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + """ + Unpacks a tensor of quantized weights that were stored in a packed format using 2 bits per value. + + Parameters: + ----------- + packed : torch.Tensor + A tensor containing packed weights where each element represents 4 quantized values (using 2 bits per value). + dtype : torch.dtype + The dtype of the returned Tensor + Returns: + -------- + torch.Tensor + A tensor of unpacked weights, where each value is converted from its packed 2-bit representation. + + Example: + -------- + packed = torch.tensor([[0b10100001, 0b00011000], + [0b10010000, 0b00001010]], dtype=torch.uint8) + + # Unpack the values + unpacked = unpack_weights(packed) + + # Resulting unpacked tensor + print(unpacked) + # Output: tensor([[ 0, -1], + [-1, 1], + [-1, 1], + [-1, 1], + [ 1, 0], + [ 0, -1], + [ 1, -1], + [ 1, -1]]) + + Explanation of the example: + --------------------------- + Let's take the first value for example 0b10100001, we we will only focus on the first column, + because every element is unpacked across the first dimension + - First 2 bits: `01` → 0 at [0][0] + - Second 2 bits: `00` → -1 at [0][2] + - Third 2 bits: `10` → 1 at [0][4] + - Fourth 2 bits: `10` → 1 at [0][6] + the second value of the same row (0b10010000) will give the values for [0][1], [0][3], [0][5], [0][7] + + We subtract 1 because during the packing process, it's easier to work with values like 0, 1, and 2. To make this possible, + we add 1 to the original ternary weights (which are typically -1, 0, and 1) when packing them. When unpacking, we reverse + this by subtracting 1 to restore the original ternary values. + """ + packed_shape = packed.shape + + if len(packed_shape) == 1: + original_row_dim = packed_shape[0] * VALUES_PER_ITEM + unpacked_shape = (original_row_dim,) + else: + original_row_dim = packed_shape[0] * VALUES_PER_ITEM + unpacked_shape = (original_row_dim, *packed_shape[1:]) + + unpacked = torch.zeros(unpacked_shape, device=packed.device, dtype=torch.uint8) + + for i in range(VALUES_PER_ITEM): + start = i * packed_shape[0] + end = start + packed_shape[0] + mask = 3 << (2 * i) + unpacked[start:end] = (packed & mask) >> (2 * i) + + return unpacked.to(dtype) - 1 + + +class BitLinear(nn.Module): + def __init__(self, in_features: int, out_features: int, bias: bool, device=None, dtype=None): + super().__init__() + self.dtype = dtype + self.in_features = in_features + self.out_features = out_features + self.register_buffer( + "weight", + torch.zeros( + (out_features // VALUES_PER_ITEM, in_features), + dtype=torch.uint8, + device=device, + ), + ) + self.register_buffer( + "weight_scale", + torch.ones( + (1), + dtype=dtype, + device=device, + ), + ) + if bias: + self.register_buffer("bias", torch.zeros((out_features), dtype=dtype, device=device)) + else: + self.bias = None + + @torch.compile + def activation_quant(self, input, num_bits=8): + """ + Activation function : Performs symmetric, per-token quantization on the input activations. + Parameters: + ----------- + x : torch.Tensor + Input activations to be quantized. + num_bits : int, optional (default=8) + Number of bits to use for quantization, determining the quantization range. + + Returns: + -------- + result : torch.Tensor + Quantized activation tensor, with values mapped to an `int8` range. + scale : torch.Tensor + The per-channel scaling factors used to quantize the tensor. + """ + Qn = -(2 ** (num_bits - 1)) + Qp = 2 ** (num_bits - 1) - 1 + scale = Qp / input.abs().max(dim=-1, keepdim=True).values.clamp(min=1e-5) + result = (input * scale).round().clamp(Qn, Qp) + return result.to(torch.int8), scale + + @torch.compile + def post_quant_process(self, input, input_scale, weight_scale): + out = input / (input_scale * weight_scale) + return out + + def forward(self, input): + w = self.weight + w_quant = unpack_weights(w, dtype=self.dtype) + input_quant, input_scale = self.activation_quant(input) + y = F.linear(input_quant.to(self.dtype), w_quant) + y = self.post_quant_process(y, self.weight_scale, input_scale) + if self.bias is not None: + y += self.bias.view(1, -1).expand_as(y) + return y + + +def _replace_with_bitnet_linear( + model, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + has_been_replaced=False, + pre_quantized=False, +): + """ + Private method that wraps the recursion for module replacement. + + Returns the converted model and a boolean that indicates if the conversion has been successfull or not. + """ + + if current_key_name is None: + current_key_name = [] + + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + # Check if the current key is not in the `modules_to_not_convert` + if not any(key in ".".join(current_key_name) for key in modules_to_not_convert): + with init_empty_weights(): + if isinstance(module, nn.Linear) and name not in modules_to_not_convert: + in_features = module.in_features + out_features = module.out_features + model._modules[name] = BitLinear( + in_features=in_features, + out_features=out_features, + bias=module.bias is not None, + device=module.weight.device, + dtype=module.weight.dtype, + ) + has_been_replaced = True + model._modules[name].requires_grad_(False) + + if len(list(module.children())) > 0: + _, has_been_replaced = _replace_with_bitnet_linear( + module, + modules_to_not_convert=modules_to_not_convert, + current_key_name=current_key_name, + quantization_config=quantization_config, + has_been_replaced=has_been_replaced, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def replace_with_bitnet_linear( + model, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + pre_quantized=False, +): + """ + A helper function to replace all `torch.nn.Linear` modules by `BitLinear158` modules`. + + The function will be run recursively and replace all `torch.nn.Linear` modules except for the `lm_head` that should + be kept as a `torch.nn.Linear` module. The replacement is done under `init_empty_weights` context manager so no + CPU/GPU memory is required to run this function. Each weight will be quantized along the channel. + + Parameters: + model (`torch.nn.Module`): + Input model or `torch.nn.Module` as the function is run recursively. + modules_to_not_convert (`List[`str`]`, *optional*, defaults to `["lm_head"]`): + Names of the modules to not convert in `EetqLinear`. In practice we keep the `lm_head` in full precision + for numerical stability reasons. + current_key_name (`List[`str`]`, *optional*): + An array to track the current key of the recursion. This is used to check whether the current key (part of + it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or + `disk`). + """ + modules_to_not_convert = ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert + if quantization_config and quantization_config.modules_to_not_convert is not None: + modules_to_not_convert.extend(quantization_config.modules_to_not_convert) + modules_to_not_convert = list(set(modules_to_not_convert)) + model, has_been_replaced = _replace_with_bitnet_linear( + model, + modules_to_not_convert, + current_key_name, + quantization_config, + pre_quantized=pre_quantized, + ) + + if not has_been_replaced: + logger.warning( + "You are loading your model using bitnet but no linear modules were found in your model." + " Please double check your model architecture, or submit an issue on github if you think this is" + " a bug." + ) + + return model diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/bitsandbytes.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/bitsandbytes.py new file mode 100644 index 0000000000000000000000000000000000000000..3973dc58c062bc9b1755a24c708e39e1ad4016ce --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/bitsandbytes.py @@ -0,0 +1,561 @@ +import importlib.metadata +import inspect +import warnings +from copy import deepcopy +from inspect import signature + +from packaging import version + +from ..utils import ( + get_available_devices, + is_accelerate_available, + is_bitsandbytes_available, + is_bitsandbytes_multi_backend_available, + is_ipex_available, + is_torch_available, + logging, +) + + +if is_bitsandbytes_available(): + import bitsandbytes as bnb + import torch + import torch.nn as nn + + from ..pytorch_utils import Conv1D + +if is_accelerate_available(): + import accelerate + from accelerate import init_empty_weights + from accelerate.hooks import add_hook_to_module, remove_hook_from_module + from accelerate.utils import find_tied_parameters + +logger = logging.get_logger(__name__) + + +def set_module_quantized_tensor_to_device(module, tensor_name, device, value=None, quantized_stats=None): + """ + A helper function to set a given tensor (parameter of buffer) of a module on a specific device (note that doing + `param.to(device)` creates a new tensor not linked to the parameter, which is why we need this function). The + function is adapted from `set_module_tensor_to_device` function from accelerate that is adapted to support the + class `Int8Params` from `bitsandbytes`. + + Args: + module (`torch.nn.Module`): + The module in which the tensor we want to move lives. + tensor_name (`str`): + The full name of the parameter/buffer. + device (`int`, `str` or `torch.device`): + The device on which to set the tensor. + value (`torch.Tensor`, *optional*): + The value of the tensor (useful when going from the meta device to any other device). + quantized_stats (`dict[str, Any]`, *optional*): + Dict with items for either 4-bit or 8-bit serialization + """ + # Recurse if needed + if "." in tensor_name: + splits = tensor_name.split(".") + for split in splits[:-1]: + new_module = getattr(module, split) + if new_module is None: + raise ValueError(f"{module} has no attribute {split}.") + module = new_module + tensor_name = splits[-1] + + if tensor_name not in module._parameters and tensor_name not in module._buffers: + raise ValueError(f"{module} does not have a parameter or a buffer named {tensor_name}.") + is_buffer = tensor_name in module._buffers + old_value = getattr(module, tensor_name) + + if old_value.device == torch.device("meta") and device not in ["meta", torch.device("meta")] and value is None: + raise ValueError(f"{tensor_name} is on the meta device, we need a `value` to put in on {device}.") + + prequantized_loading = quantized_stats is not None + if is_buffer or not is_bitsandbytes_available(): + is_8bit = False + is_4bit = False + else: + is_4bit = hasattr(bnb.nn, "Params4bit") and isinstance(module._parameters[tensor_name], bnb.nn.Params4bit) + is_8bit = isinstance(module._parameters[tensor_name], bnb.nn.Int8Params) + + if is_8bit or is_4bit: + param = module._parameters[tensor_name] + if param.device.type != "cuda": + if value is None: + new_value = old_value.to(device) + elif isinstance(value, torch.Tensor): + new_value = value.to("cpu") + else: + new_value = torch.tensor(value, device="cpu") + + # Support models using `Conv1D` in place of `nn.Linear` (e.g. openai-community/gpt2) by transposing the weight matrix prior to quantization. + # Since weights are saved in the correct "orientation", we skip transposing when loading. + if issubclass(module.source_cls, Conv1D) and not prequantized_loading: + new_value = new_value.T + + kwargs = old_value.__dict__ + + if prequantized_loading != (new_value.dtype in (torch.int8, torch.uint8)): + raise ValueError( + f"Value dtype `{new_value.dtype}` is not compatible with parameter quantization status." + ) + + if is_8bit: + is_8bit_serializable = version.parse(importlib.metadata.version("bitsandbytes")) > version.parse( + "0.37.2" + ) + if new_value.dtype in (torch.int8, torch.uint8) and not is_8bit_serializable: + raise ValueError( + "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. " + "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`." + ) + new_value = bnb.nn.Int8Params(new_value, requires_grad=False, **kwargs).to(device) + if prequantized_loading: + setattr(new_value, "SCB", quantized_stats["SCB"].to(device)) + elif is_4bit: + if prequantized_loading: + is_4bit_serializable = version.parse(importlib.metadata.version("bitsandbytes")) >= version.parse( + "0.41.3" + ) + if new_value.dtype in (torch.int8, torch.uint8) and not is_4bit_serializable: + raise ValueError( + "Detected 4-bit weights but the version of bitsandbytes is not compatible with 4-bit serialization. " + "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`." + ) + new_value = bnb.nn.Params4bit.from_prequantized( + data=new_value, + quantized_stats=quantized_stats, + requires_grad=False, + device=device, + **kwargs, + ) + else: + new_value = bnb.nn.Params4bit(new_value, requires_grad=False, **kwargs).to(device) + module._parameters[tensor_name] = new_value + + else: + if value is None: + new_value = old_value.to(device) + elif isinstance(value, torch.Tensor): + new_value = value.to(device) + else: + new_value = torch.tensor(value, device=device) + + if is_buffer: + module._buffers[tensor_name] = new_value + else: + new_value = nn.Parameter(new_value, requires_grad=old_value.requires_grad) + module._parameters[tensor_name] = new_value + + +def _replace_with_bnb_linear( + model, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + has_been_replaced=False, +): + """ + Private method that wraps the recursion for module replacement. + + Returns the converted model and a boolean that indicates if the conversion has been successfull or not. + """ + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + if (isinstance(module, nn.Linear) or isinstance(module, Conv1D)) and name not in modules_to_not_convert: + # Check if the current key is not in the `modules_to_not_convert` + current_key_name_str = ".".join(current_key_name) + if not any( + (key + "." in current_key_name_str) or (key == current_key_name_str) for key in modules_to_not_convert + ): + with init_empty_weights(): + if isinstance(module, Conv1D): + in_features, out_features = module.weight.shape + else: + in_features = module.in_features + out_features = module.out_features + + if quantization_config.quantization_method() == "llm_int8": + model._modules[name] = bnb.nn.Linear8bitLt( + in_features, + out_features, + module.bias is not None, + has_fp16_weights=quantization_config.llm_int8_has_fp16_weight, + threshold=quantization_config.llm_int8_threshold, + ) + has_been_replaced = True + else: + if ( + quantization_config.llm_int8_skip_modules is not None + and name in quantization_config.llm_int8_skip_modules + ): + pass + else: + extra_kwargs = ( + {"quant_storage": quantization_config.bnb_4bit_quant_storage} + if "quant_storage" in list(signature(bnb.nn.Linear4bit).parameters) + else {} + ) + model._modules[name] = bnb.nn.Linear4bit( + in_features, + out_features, + module.bias is not None, + quantization_config.bnb_4bit_compute_dtype, + compress_statistics=quantization_config.bnb_4bit_use_double_quant, + quant_type=quantization_config.bnb_4bit_quant_type, + **extra_kwargs, + ) + has_been_replaced = True + # Store the module class in case we need to transpose the weight later + model._modules[name].source_cls = type(module) + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) + if len(list(module.children())) > 0: + _, has_been_replaced = _replace_with_bnb_linear( + module, + modules_to_not_convert, + current_key_name, + quantization_config, + has_been_replaced=has_been_replaced, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def replace_with_bnb_linear(model, modules_to_not_convert=None, current_key_name=None, quantization_config=None): + """ + A helper function to replace all `torch.nn.Linear` modules by `bnb.nn.Linear8bit` modules from the `bitsandbytes` + library. This will enable running your models using mixed int8 precision as described by the paper `LLM.int8(): + 8-bit Matrix Multiplication for Transformers at Scale`. Make sure `bitsandbytes` compiled with the correct CUDA + version of your hardware is installed before running this function. `pip install -i https://test.pypi.org/simple/ + bitsandbytes` + + The function will be run recursively and replace all `torch.nn.Linear` modules except for the `lm_head` that should + be kept as a `torch.nn.Linear` module. The replacement is done under `init_empty_weights` context manager so no + CPU/GPU memory is required to run this function. Int8 mixed-precision matrix decomposition works by separating a + matrix multiplication into two streams: (1) and systematic feature outlier stream matrix multiplied in fp16 + (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no + predictive degradation is possible for very large models (>=176B parameters). + + Parameters: + model (`torch.nn.Module`): + Input model or `torch.nn.Module` as the function is run recursively. + modules_to_not_convert (`List[`str`]`, *optional*, defaults to `["lm_head"]`): + Names of the modules to not convert in `Linear8bitLt`. In practice we keep the `lm_head` in full precision + for numerical stability reasons. + current_key_name (`List[`str`]`, *optional*): + An array to track the current key of the recursion. This is used to check whether the current key (part of + it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or + `disk`). + quantization_config ('transformers.utils.quantization_config.BitsAndBytesConfig'): + To configure and manage settings related to quantization, a technique used to compress neural network models + by reducing the precision of the weights and activations, thus making models more efficient in terms of both + storage and computation. + """ + modules_to_not_convert = ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert + model, has_been_replaced = _replace_with_bnb_linear( + model, modules_to_not_convert, current_key_name, quantization_config + ) + + if not has_been_replaced: + logger.warning( + "You are loading your model in 8bit or 4bit but no linear modules were found in your model." + " Please double check your model architecture, or submit an issue on github if you think this is" + " a bug." + ) + + return model + + +# For backward compatibility +def replace_8bit_linear(*args, **kwargs): + warnings.warn( + "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead", + FutureWarning, + ) + return replace_with_bnb_linear(*args, **kwargs) + + +# For backward compatiblity +def set_module_8bit_tensor_to_device(*args, **kwargs): + warnings.warn( + "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead", + FutureWarning, + ) + return set_module_quantized_tensor_to_device(*args, **kwargs) + + +def get_keys_to_not_convert(model): + r""" + An utility function to get the key of the module to keep in full precision if any For example for CausalLM modules + we may want to keep the lm_head in full precision for numerical stability reasons. For other architectures, we want + to keep the tied weights of the model. The function will return a list of the keys of the modules to not convert in + int8. + + Parameters: + model (`torch.nn.Module`): + Input model + """ + # Create a copy of the model and tie the weights, then + # check if it contains tied weights + tied_model = deepcopy(model) # this has 0 cost since it is done inside `init_empty_weights` context manager` + tied_model.tie_weights() + + tied_params = find_tied_parameters(tied_model) + # For compatibility with Accelerate < 0.18 + if isinstance(tied_params, dict): + tied_keys = sum(list(tied_params.values()), []) + list(tied_params.keys()) + else: + tied_keys = sum(tied_params, []) + has_tied_params = len(tied_keys) > 0 + + # If there is not tied weights, we want to keep the lm_head（output_embedding) in full precision + if not has_tied_params: + output_emb = model.get_output_embeddings() + if output_emb is not None: + list_last_module = [name for name, module in model.named_modules() if id(module) == id(output_emb)] + return list_last_module + + # otherwise, no tied weights, no output embedding defined, simply keep the last module in full precision + list_modules = list(model.named_parameters()) + list_last_module = [list_modules[-1][0]] + # add last module together with tied weights + intersection = set(list_last_module) - set(tied_keys) + list_untouched = list(set(tied_keys)) + list(intersection) + + # remove ".weight" from the keys + names_to_remove = [".weight", ".bias"] + filtered_module_names = [] + for name in list_untouched: + for name_to_remove in names_to_remove: + if name_to_remove in name: + name = name.replace(name_to_remove, "") + filtered_module_names.append(name) + + return filtered_module_names + + +# Copied from PEFT: https://github.com/huggingface/peft/blob/47b3712898539569c02ec5b3ed4a6c36811331a1/src/peft/utils/integrations.py#L41 +def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", state=None): + """ + Helper function to dequantize 4bit or 8bit bnb weights. + + If the weight is not a bnb quantized weight, it will be returned as is. + """ + if not isinstance(weight, torch.nn.Parameter): + raise TypeError(f"Input weight should be of type nn.Parameter, got {type(weight)} instead") + + cls_name = weight.__class__.__name__ + if cls_name not in ("Params4bit", "Int8Params"): + return weight + + if cls_name == "Params4bit": + output_tensor = bnb.functional.dequantize_4bit(weight.data, weight.quant_state) + logger.warning_once( + f"The model is going to be dequantized in {output_tensor.dtype} - if you want to upcast it to another dtype, make sure to pass the desired dtype when quantizing the model through `bnb_4bit_quant_type` argument of `BitsAndBytesConfig`" + ) + return output_tensor.to(dtype) + + if state.SCB is None: + state.SCB = weight.SCB + + if hasattr(bnb.functional, "int8_vectorwise_dequant"): + # Use bitsandbytes API if available (requires v0.45.0+) + dequantized = bnb.functional.int8_vectorwise_dequant(weight.data, state.SCB) + else: + # Multiply by (scale/127) to dequantize. + dequantized = weight.data * state.SCB.view(-1, 1) * 7.874015718698502e-3 + + return dequantized.to(dtype) + + +def _create_accelerate_new_hook(old_hook): + r""" + Creates a new hook based on the old hook. Use it only if you know what you are doing ! + This method is a copy of: https://github.com/huggingface/peft/blob/748f7968f3a31ec06a1c2b0328993319ad9a150a/src/peft/utils/other.py#L245 + with some changes + """ + old_hook_cls = getattr(accelerate.hooks, old_hook.__class__.__name__) + old_hook_attr = old_hook.__dict__ + filtered_old_hook_attr = {} + old_hook_init_signature = inspect.signature(old_hook_cls.__init__) + for k in old_hook_attr.keys(): + if k in old_hook_init_signature.parameters: + filtered_old_hook_attr[k] = old_hook_attr[k] + new_hook = old_hook_cls(**filtered_old_hook_attr) + return new_hook + + +def _dequantize_and_replace( + model, + dtype, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + has_been_replaced=False, +): + """ + Converts a quantized model into its dequantized original version. The newly converted model will have + some performance drop compared to the original model before quantization - use it only for specific usecases + such as QLoRA adapters merging. + + Returns the converted model and a boolean that indicates if the conversion has been successfull or not. + """ + quant_method = quantization_config.quantization_method() + + target_cls = bnb.nn.Linear8bitLt if quant_method == "llm_int8" else bnb.nn.Linear4bit + + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + if isinstance(module, target_cls) and name not in modules_to_not_convert: + # Check if the current key is not in the `modules_to_not_convert` + current_key_name_str = ".".join(current_key_name) + + if not any( + (key + "." in current_key_name_str) or (key == current_key_name_str) for key in modules_to_not_convert + ): + bias = getattr(module, "bias", None) + + device = module.weight.device + with init_empty_weights(): + new_module = torch.nn.Linear(module.in_features, module.out_features, bias=bias is not None) + + if quant_method == "llm_int8": + state = module.state + else: + state = None + + new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, dtype, state)) + + if bias is not None: + new_module.bias = bias + + # Create a new hook and attach it in case we use accelerate + if hasattr(module, "_hf_hook"): + old_hook = module._hf_hook + new_hook = _create_accelerate_new_hook(old_hook) + + remove_hook_from_module(module) + add_hook_to_module(new_module, new_hook) + + new_module.to(device) + model._modules[name] = new_module + has_been_replaced = True + if len(list(module.children())) > 0: + _, has_been_replaced = _dequantize_and_replace( + module, + dtype, + modules_to_not_convert, + current_key_name, + quantization_config, + has_been_replaced=has_been_replaced, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def dequantize_and_replace( + model, + modules_to_not_convert=None, + quantization_config=None, +): + model, has_been_replaced = _dequantize_and_replace( + model, + model.dtype, + modules_to_not_convert=modules_to_not_convert, + quantization_config=quantization_config, + ) + + if not has_been_replaced: + logger.warning( + "For some reason the model has not been properly dequantized. You might see unexpected behavior." + ) + + return model + + +def _validate_bnb_multi_backend_availability(raise_exception): + import bitsandbytes as bnb + + bnb_supported_devices = getattr(bnb, "supported_torch_devices", set()) + available_devices = set(get_available_devices()) + + if available_devices == {"cpu"} and not is_ipex_available(): + from importlib.util import find_spec + + if find_spec("intel_extension_for_pytorch"): + logger.warning( + "You have Intel IPEX installed but if you're intending to use it for CPU, it might not have the right version. Be sure to double check that your PyTorch and IPEX installs are compatible." + ) + + available_devices = frozenset( + [device for device in available_devices if device != "cpu"] + ) # Only Intel CPU is supported by BNB at the moment + + if not available_devices.intersection(bnb_supported_devices): + if raise_exception: + bnb_supported_devices_with_info = set( # noqa: C401 + '"cpu" (needs an Intel CPU and intel_extension_for_pytorch installed and compatible with the PyTorch version)' + if device == "cpu" + else device + for device in bnb_supported_devices + ) + err_msg = ( + f"None of the available devices `available_devices = {available_devices or None}` are supported by the bitsandbytes version you have installed: `bnb_supported_devices = {bnb_supported_devices_with_info}`. " + "Please check the docs to see if the backend you intend to use is available and how to install it: https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend" + ) + + logger.error(err_msg) + raise RuntimeError(err_msg) + + logger.warning("No supported devices found for bitsandbytes multi-backend.") + return False + + logger.debug("Multi-backend validation successful.") + return True + + +def _validate_bnb_cuda_backend_availability(raise_exception): + if not is_torch_available(): + return False + + import torch + + if not torch.cuda.is_available(): + log_msg = ( + "CUDA is required but not available for bitsandbytes. Please consider installing the multi-platform enabled version of bitsandbytes, which is currently a work in progress. " + "Please check currently supported platforms and installation instructions at https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend" + ) + if raise_exception: + logger.error(log_msg) + raise RuntimeError(log_msg) + + logger.warning(log_msg) + return False + + logger.debug("CUDA backend validation successful.") + return True + + +def validate_bnb_backend_availability(raise_exception=False): + """ + Validates if the available devices are supported by bitsandbytes, optionally raising an exception if not. + """ + if not is_bitsandbytes_available(): + if importlib.util.find_spec("bitsandbytes") and version.parse( + importlib.metadata.version("bitsandbytes") + ) < version.parse("0.43.1"): + return _validate_bnb_cuda_backend_availability(raise_exception) + return False + + if is_bitsandbytes_multi_backend_available(): + return _validate_bnb_multi_backend_availability(raise_exception) + return _validate_bnb_cuda_backend_availability(raise_exception) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/cache_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/cache_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8db9eeec9d38598650c142bb0f1d3022339f06b2 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/cache_utils.py @@ -0,0 +1,2374 @@ +import copy +import importlib.metadata +import json +import os +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import torch +from packaging import version + +from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_6 + +from .configuration_utils import PretrainedConfig +from .utils import is_hqq_available, is_optimum_quanto_available, is_torch_greater_or_equal, logging + + +if is_hqq_available(): + from hqq.core.quantize import Quantizer as HQQQuantizer + +logger = logging.get_logger(__name__) + + +class Cache: + """ + Base, abstract class for all caches. The actual data structure is specific to each subclass. + """ + + is_compileable = False + + def __init__(self): + super().__init__() + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. These are specific to each subclass and allow new types of + cache to be created. + + Return: + A tuple containing the updated key and value states. + """ + raise NotImplementedError("Make sure to implement `update` in a subclass.") + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # TODO: deprecate this function in favor of `cache_position` + raise NotImplementedError("Make sure to implement `get_seq_length` in a subclass.") + + def get_max_cache_shape(self) -> Optional[int]: + """Returns the maximum sequence length (i.e. max capacity) of the cache object""" + raise NotImplementedError("Make sure to implement `get_max_cache_shape` in a subclass.") + + def get_usable_length(self, new_seq_length: int, layer_idx: Optional[int] = 0) -> int: + """Given the sequence length of the new inputs, returns the usable length of the cache.""" + # Cache without size limit -> all cache is usable + # Cache with size limit -> if the length cache plus the length of the new inputs is larger the maximum cache + # length, we will need to evict part of the cache (and thus not all cache is usable) + max_length = self.get_max_cache_shape() + previous_seq_length = self.get_seq_length(layer_idx) + if max_length is not None and previous_seq_length + new_seq_length > max_length: + return max_length - new_seq_length + return previous_seq_length + + def reorder_cache(self, beam_idx: torch.LongTensor): + """Reorders the cache for beam search, given the selected beam indices.""" + for layer_idx in range(len(self.key_cache)): + if self.key_cache[layer_idx].numel(): + device = self.key_cache[layer_idx].device + self.key_cache[layer_idx] = self.key_cache[layer_idx].index_select(0, beam_idx.to(device)) + if self.value_cache[layer_idx].numel(): + device = self.value_cache[layer_idx].device + self.value_cache[layer_idx] = self.value_cache[layer_idx].index_select(0, beam_idx.to(device)) + + @property + def seen_tokens(self): + logger.warning_once( + "The `seen_tokens` attribute is deprecated and will be removed in v4.41. Use the `cache_position` " + "model input instead." + ) + if hasattr(self, "_seen_tokens"): + return self._seen_tokens + else: + return None + + +@dataclass +class CacheConfig: + """ + Base class for cache configs + """ + + cache_implementation: None + + @classmethod + def from_dict(cls, config_dict, **kwargs): + """ + Constructs a CacheConfig instance from a dictionary of parameters. + Args: + config_dict (Dict[str, Any]): Dictionary containing configuration parameters. + **kwargs: Additional keyword arguments to override dictionary values. + + Returns: + CacheConfig: Instance of CacheConfig constructed from the dictionary. + """ + config = cls(**config_dict) + to_remove = [] + for key, value in kwargs.items(): + if hasattr(config, key): + setattr(config, key, value) + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + return config + + # Copied from transformers.utils.quantization_config.QuantizationConfigMixin.to_json_file + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this configuration instance's parameters will be saved. + use_diff (`bool`, *optional*, defaults to `True`): + If set to `True`, only the difference between the config instance and the default + `QuantizationConfig()` is serialized to JSON file. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + config_dict = self.to_dict() + json_string = json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + writer.write(json_string) + + # Copied from transformers.utils.quantization_config.QuantizationConfigMixin.to_dict + def to_dict(self) -> Dict[str, Any]: + """ + Serializes this instance to a Python dictionary. Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + return copy.deepcopy(self.__dict__) + + # Copied from transformers.utils.quantization_config.QuantizationConfigMixin.__iter__ + def __iter__(self): + """allows `dict(obj)` for situations where obj may be a dict or QuantizationConfigMixin""" + for attr, value in copy.deepcopy(self.__dict__).items(): + yield attr, value + + # Copied from transformers.utils.quantization_config.QuantizationConfigMixin.__repr__ + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + def to_json_string(self): + """ + Serializes this instance to a JSON formatted string. + Returns: + str: JSON formatted string representing the configuration instance. + """ + return json.dumps(self.__dict__, indent=2) + "\n" + + # Copied from transformers.utils.quantization_config.QuantizationConfigMixin.update + def update(self, **kwargs): + """ + Updates attributes of this class instance with attributes from `kwargs` if they match existing attributes, + returning all the unused kwargs. + + Args: + kwargs (`Dict[str, Any]`): + Dictionary of attributes to tentatively update this class. + + Returns: + `Dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance. + """ + to_remove = [] + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + to_remove.append(key) + + # Remove all the attributes that were updated, without modifying the input dict + unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove} + return unused_kwargs + + +@dataclass +class QuantizedCacheConfig(CacheConfig): + """ + Configuration class for quantized cache settings. + + Attributes: + backend (`str`, *optional*, defaults to `"quanto"`): + Backend to use when performing quantization, Can be one of [`quanto`, `HQQ`] + nbits (`Optional[int]`, *optional*, defaults to 4): + Number of bits, can be 2 or 4 for the `quanto` backend and one of [1, 2, 3, 4, 8] for the `HQQ` backend. Defaults to 2. + axis_key (`int`, *optional*, defaults to 0): + Axis over which to perform grouping for the key tensors. Can be [0, -1] for `quanto` backend and [0, 1] for `HQQ` backend. + axis_value (`int`, *optional*, defaults to 0): + Axis over which to perform grouping for the value tensors. Can be [0, -1] for `quanto` backend and [0, 1] for `HQQ` backend. + q_group_size (`Optional[int]`, *optional*, defaults to 64): + Size of the quantization group, should be a divisor of the model's hidden dimension. + Defaults to 64. + residual_length (`Optional[int]`, *optional*, defaults to 128): + Length of the residual cache which will always be stored in original precision. + Defaults to 128. + compute_dtype (`torch.dtype`, *optional*, defaults to `torch.float16`): + The default dtype used for computations in the model. Keys and Values will be cast to this dtype after dequantization. + device (`str`, *optional*, defaults to `"cpu"`): + Device on which to perform computations, should be same as the model's device. + """ + + def __init__( + self, + backend: str = "quanto", + nbits: Optional[int] = 4, + axis_key: Optional[int] = 0, + axis_value: Optional[int] = 0, + q_group_size: Optional[int] = 64, + residual_length: Optional[int] = 128, + compute_dtype: Optional[torch.dtype] = torch.float16, + device: Optional[str] = "cpu", + ): + self.backend = backend + self.nbits = nbits + self.axis_key = axis_key + self.axis_value = axis_value + self.q_group_size = q_group_size + self.residual_length = residual_length + self.compute_dtype = compute_dtype + self.device = device + + def validate(self): + """Validates if the arguments passed are correct""" + + incorrect_arg_msg = ( + "Some of the keys in `cache_config` are defined incorrectly. `{key}` should be {correct_value}` " + "but found {found_value}" + ) + # Check that the values are reasonable in general (nbits, axis) + # Later in QuantizedCache init we check if they are supported for that particular backend + if self.nbits not in [1, 2, 3, 4, 8]: + raise ValueError( + incorrect_arg_msg.format( + key="nbits", + correct_value="2 or 4 or 8", + found_value=self.nbits, + ), + ) + if self.q_group_size <= 0: + raise ValueError( + incorrect_arg_msg.format( + key="q_group_size", + correct_value="a positive integer", + found_value=self.q_group_size, + ), + ) + if self.residual_length < 0: + raise ValueError( + incorrect_arg_msg.format( + key="residual_length", + correct_value="a positive integer", + found_value=self.residual_length, + ), + ) + + if self.axis_key not in [0, 1, -1]: + raise ValueError( + incorrect_arg_msg.format( + key="axis_key", + correct_value="`1` or `0`, `-1`", + found_value=self.axis_key, + ), + ) + + if self.axis_value not in [0, 1, -1]: + raise ValueError( + incorrect_arg_msg.format( + key="axis_value", + correct_value="`1` or `0` or `-1`", + found_value=self.axis_value, + ), + ) + + +@dataclass +class StaticCacheConfig(CacheConfig): + """ + Configuration class for static cache settings. + """ + + cache_implementation = "static" + + def __init__(self, batch_size: int, max_cache_len: int, device="cpu"): + self.batch_size = batch_size + self.max_cache_len = max_cache_len + self.device = device + + def validate(self): + """Validates if the arguments passed are correct""" + + incorrect_arg_msg = ( + "Some of the keys in `cache_config` are defined incorrectly. `{key}` should be {correct_value}` " + "but found {found_value}" + ) + + if self.batch_size <= 0: + raise ValueError( + incorrect_arg_msg.format( + key="batch_size", + correct_value="> 0", + found_value=self.batch_size, + ), + ) + + if self.max_cache_len <= 0: + raise ValueError( + incorrect_arg_msg.format( + key="max_cache_len", + correct_value="> 0", + found_value=self.max_cache_len, + ), + ) + + +class DynamicCache(Cache): + """ + A cache that grows dynamically as more tokens are generated. This is the default for generative models. + + It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is + `[batch_size, num_heads, seq_len, head_dim]`. + + Example: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache + + >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + + >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> past_key_values = DynamicCache() + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + DynamicCache() + ``` + """ + + def __init__(self, _distributed_cache_data: Iterable = None) -> None: + super().__init__() + self._seen_tokens = 0 # Used in `generate` to keep tally of how many tokens the cache has seen + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + + # `_distributed_cache_data` was originally added for compatibility with `torch.distributed` (DDP). See #36121 + # and #36373 for more information. In a nutshell, it is `map(gather_map, zip(*caches))`, i.e. each item in the + # iterable contains the key and value states for a layer gathered across replicas by torch.distributed + # (shape=[global batch size, num_heads, seq_len, head_dim]). + # WARNING: `_distributed_cache_data` must be the first argument in `__init__`, otherwise we'll break + # compatibility. The name of the argument doesn't matter. + if _distributed_cache_data is not None: + for key_states, value_states in _distributed_cache_data: + self.key_cache.append(key_states) + self.value_cache.append(value_states) + + def __getitem__(self, layer_idx: int) -> List[Tuple[torch.Tensor]]: + """ + Support for backwards-compatible `past_key_value` indexing, e.g. `past_key_value[0][0].shape[2]` to get the + sequence length. + """ + if layer_idx < len(self): + return (self.key_cache[layer_idx], self.value_cache[layer_idx]) + else: + raise KeyError(f"Cache only has {len(self)} layers, attempted to access layer with index {layer_idx}") + + def __iter__(self): + """ + Support for backwards-compatible `past_key_value` iteration, e.g. `for x in past_key_value:` to iterate over + keys and values + """ + for layer_idx in range(len(self)): + yield (self.key_cache[layer_idx], self.value_cache[layer_idx]) + + def __len__(self): + """ + Support for backwards-compatible `past_key_value` length, e.g. `len(past_key_value)`. This value corresponds + to the number of layers in the model. + """ + return len(self.key_cache) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. No additional arguments are used in `DynamicCache`. + + Return: + A tuple containing the updated key and value states. + """ + # Update the number of seen tokens + if layer_idx == 0: + self._seen_tokens += key_states.shape[-2] + + # Update the cache + if key_states is not None: + if len(self.key_cache) <= layer_idx: + # There may be skipped layers, fill them with empty lists + for _ in range(len(self.key_cache), layer_idx): + self.key_cache.append(torch.tensor([])) + self.value_cache.append(torch.tensor([])) + self.key_cache.append(key_states) + self.value_cache.append(value_states) + elif ( + not self.key_cache[layer_idx].numel() # prefers not t.numel() to len(t) == 0 to export the model + ): # fills previously skipped layers; checking for tensor causes errors + self.key_cache[layer_idx] = key_states + self.value_cache[layer_idx] = value_states + else: + self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=-2) + self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=-2) + + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # TODO: deprecate this function in favor of `cache_position` + is_empty_layer = ( + len(self.key_cache) == 0 # no cache in any layer + or len(self.key_cache) <= layer_idx # skipped `layer_idx` and hasn't run a layer with cache after it + or not self.key_cache[layer_idx].numel() # the layer has no cache + ) + layer_seq_length = self.key_cache[layer_idx].shape[-2] if not is_empty_layer else 0 + return layer_seq_length + + def get_max_cache_shape(self) -> Optional[int]: + """Returns the maximum sequence length of the cache object. DynamicCache does not have a maximum length.""" + return None + + def to_legacy_cache(self) -> Tuple[Tuple[torch.Tensor], Tuple[torch.Tensor]]: + """Converts the `DynamicCache` instance into the its equivalent in the legacy cache format. Used for + backward compatibility.""" + legacy_cache = () + for layer_idx in range(len(self)): + legacy_cache += ((self.key_cache[layer_idx], self.value_cache[layer_idx]),) + return legacy_cache + + @classmethod + def from_legacy_cache(cls, past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None) -> "DynamicCache": + """Converts a cache in the legacy cache format into an equivalent `DynamicCache`. Used for + backward compatibility.""" + cache = cls() + if past_key_values is not None: + for layer_idx in range(len(past_key_values)): + key_states, value_states = past_key_values[layer_idx] + cache.update(key_states, value_states, layer_idx) + return cache + + def crop(self, max_length: int): + """Crop the past key values up to a new `max_length` in terms of tokens. `max_length` can also be + negative to remove `max_length` tokens. This is used in assisted decoding and contrastive search.""" + # In case it is negative + if max_length < 0: + max_length = self.get_seq_length() - abs(max_length) + + if self.get_seq_length() <= max_length: + return + + self._seen_tokens = max_length + for idx in range(len(self.key_cache)): + if self.key_cache[idx].numel(): + self.key_cache[idx] = self.key_cache[idx][..., :max_length, :] + self.value_cache[idx] = self.value_cache[idx][..., :max_length, :] + + def batch_split(self, full_batch_size: int, split_size: int) -> List["DynamicCache"]: + """Split the current instance into a list of `DynamicCache` by the batch size. This will be used by + `_split_model_inputs()` in `generation.utils`""" + out = [] + for i in range(0, full_batch_size, split_size): + current_split = DynamicCache() + current_split._seen_tokens = self._seen_tokens + current_split.key_cache = [tensor[i : i + split_size] for tensor in self.key_cache] + current_split.value_cache = [tensor[i : i + split_size] for tensor in self.value_cache] + out.append(current_split) + return out + + @classmethod + def from_batch_splits(cls, splits: List["DynamicCache"]) -> "DynamicCache": + """This is the opposite of the above `batch_split()` method. This will be used by `stack_model_outputs` in + `generation.utils`""" + cache = cls() + for idx in range(len(splits[0])): + key_cache = [current.key_cache[idx] for current in splits if current.key_cache[idx].numel()] + value_cache = [current.value_cache[idx] for current in splits if current.value_cache[idx].numel()] + if key_cache != []: + layer_keys = torch.cat(key_cache, dim=0) + layer_values = torch.cat(value_cache, dim=0) + cache.update(layer_keys, layer_values, idx) + return cache + + def batch_repeat_interleave(self, repeats: int): + """Repeat the cache `repeats` times in the batch dimension. Used in contrastive search.""" + for layer_idx in range(len(self)): + self.key_cache[layer_idx] = self.key_cache[layer_idx].repeat_interleave(repeats, dim=0) + self.value_cache[layer_idx] = self.value_cache[layer_idx].repeat_interleave(repeats, dim=0) + + def batch_select_indices(self, indices: torch.Tensor): + """Only keep the `indices` in the batch dimension of the cache. Used in contrastive search.""" + for layer_idx in range(len(self)): + self.key_cache[layer_idx] = self.key_cache[layer_idx][indices, ...] + self.value_cache[layer_idx] = self.value_cache[layer_idx][indices, ...] + + +# Utilities for `DynamicCache` <> torch.export support +def _flatten_dynamic_cache( + dynamic_cache: DynamicCache, +): + """Flattens DynamicCache into flat list of tensors for `torch.export.export` to consume""" + if not isinstance(dynamic_cache, DynamicCache): + raise RuntimeError("This pytree flattening function should only be applied to DynamicCache") + + if not is_torch_greater_or_equal_than_2_6: + logger.warning_once( + "DynamicCache + torch.export is tested on torch 2.6.0+ and may not work on earlier versions." + ) + + # NOTE it seems _seen_tokens is deprecated, so probably doesn't need tracking + dictionary = { + "key_cache": getattr(dynamic_cache, "key_cache"), + "value_cache": getattr(dynamic_cache, "value_cache"), + } + return torch.utils._pytree._dict_flatten(dictionary) + + +def _flatten_with_keys_dynamic_cache(dynamic_cache: DynamicCache): + dictionary = { + "key_cache": getattr(dynamic_cache, "key_cache"), + "value_cache": getattr(dynamic_cache, "value_cache"), + } + return torch.utils._pytree._dict_flatten_with_keys(dictionary) + + +def _unflatten_dynamic_cache( + values, + context: torch.utils._pytree.Context, +): + dictionary = torch.utils._pytree._dict_unflatten(values, context) + cache = DynamicCache() + for k, v in dictionary.items(): + setattr(cache, k, v) + return cache + + +def _flatten_dynamic_cache_for_fx(cache, spec): + dictionary = { + "key_cache": getattr(cache, "key_cache"), + "value_cache": getattr(cache, "value_cache"), + } + return torch.utils._pytree.tree_flatten(dictionary)[0] + + +if is_torch_greater_or_equal("2.3"): + torch.utils._pytree.register_pytree_node( + DynamicCache, + _flatten_dynamic_cache, + _unflatten_dynamic_cache, + serialized_type_name=f"{DynamicCache.__module__}.{DynamicCache.__name__}", + flatten_with_keys_fn=_flatten_with_keys_dynamic_cache, + ) + # TODO (tmanlaibaatar) This won't be needed in torch 2.7. + torch.fx._pytree.register_pytree_flatten_spec(DynamicCache, _flatten_dynamic_cache_for_fx) + + +class OffloadedCache(DynamicCache): + """ + A drop-in replacement for DynamicCache that conserves accelerator(GPU, XPU) memory at the expense of more CPU memory. + Useful for generating from models with very long context. + + In addition to the default accelerator stream, where all forward() computations happen, + this class uses another stream, the prefetch stream, which it creates itself. + Since scheduling of operations on separate streams happens independently, this class uses + the prefetch stream to asynchronously prefetch the KV cache of layer k+1 when layer k is executing. + The movement of the layer k-1 cache to the CPU is handled by the default stream as a simple way to + ensure the eviction is scheduled after all computations on that cache are finished. + """ + + def __init__(self) -> None: + if not ( + torch.cuda.is_available() + or (is_torch_greater_or_equal("2.7", accept_dev=True) and torch.xpu.is_available()) + ): + raise RuntimeError( + "OffloadedCache can only be used with a GPU" + + (" or XPU" if is_torch_greater_or_equal("2.7", accept_dev=True) else "") + ) + + super().__init__() + self.original_device = [] + self.prefetch_stream = None + self.prefetch_stream = ( + torch.Stream() if is_torch_greater_or_equal("2.7", accept_dev=True) else torch.cuda.Stream() + ) + self.beam_idx = None # used to delay beam search operations + + def prefetch_layer(self, layer_idx: int): + "Starts prefetching the next layer cache" + if layer_idx < len(self): + with ( + self.prefetch_stream + if is_torch_greater_or_equal("2.7", accept_dev=True) + else torch.cuda.stream(self.prefetch_stream) + ): + # Prefetch next layer tensors to GPU + device = self.original_device[layer_idx] + self.key_cache[layer_idx] = self.key_cache[layer_idx].to(device, non_blocking=True) + self.value_cache[layer_idx] = self.value_cache[layer_idx].to(device, non_blocking=True) + + def evict_previous_layer(self, layer_idx: int): + "Moves the previous layer cache to the CPU" + if len(self) > 2: + # We do it on the default stream so it occurs after all earlier computations on these tensors are done + prev_layer_idx = (layer_idx - 1) % len(self) + self.key_cache[prev_layer_idx] = self.key_cache[prev_layer_idx].to("cpu", non_blocking=True) + self.value_cache[prev_layer_idx] = self.value_cache[prev_layer_idx].to("cpu", non_blocking=True) + + def __getitem__(self, layer_idx: int) -> List[Tuple[torch.Tensor]]: + "Gets the cache for this layer to the device. Prefetches the next and evicts the previous layer." + if layer_idx < len(self): + # Evict the previous layer if necessary + if is_torch_greater_or_equal("2.7", accept_dev=True): + torch.accelerator.current_stream().synchronize() + else: + torch.cuda.current_stream().synchronize() + self.evict_previous_layer(layer_idx) + # Load current layer cache to its original device if not already there + original_device = self.original_device[layer_idx] + self.prefetch_stream.synchronize() + key_tensor = self.key_cache[layer_idx] + value_tensor = self.value_cache[layer_idx] + # Now deal with beam search ops which were delayed + if self.beam_idx is not None: + self.beam_idx = self.beam_idx.to(original_device) + key_tensor = key_tensor.index_select(0, self.beam_idx) + value_tensor = value_tensor.index_select(0, self.beam_idx) + # Prefetch the next layer + self.prefetch_layer((layer_idx + 1) % len(self)) + return (key_tensor, value_tensor) + else: + raise KeyError(f"Cache only has {len(self)} layers, attempted to access layer with index {layer_idx}") + + def reorder_cache(self, beam_idx: torch.LongTensor): + """Saves the beam indices and reorders the cache when the tensor is back to its device.""" + # We delay this operation until the tensors are back to their original + # device because performing torch.index_select on the CPU is very slow + del self.beam_idx + self.beam_idx = beam_idx.clone() + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. No additional arguments are used in `OffloadedCache`. + Return: + A tuple containing the updated key and value states. + """ + # Update the number of seen tokens + if layer_idx == 0: + self._seen_tokens += key_states.shape[-2] + + # Update the cache + if len(self.key_cache) < layer_idx: + raise ValueError("OffloadedCache does not support model usage where layers are skipped. Use DynamicCache.") + elif len(self.key_cache) == layer_idx: + self.key_cache.append(key_states) + self.value_cache.append(value_states) + self.original_device.append(key_states.device) + self.evict_previous_layer(layer_idx) + else: + key_tensor, value_tensor = self[layer_idx] + self.key_cache[layer_idx] = torch.cat([key_tensor, key_states], dim=-2) + self.value_cache[layer_idx] = torch.cat([value_tensor, value_states], dim=-2) + + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + # According to https://docs.python.org/3/library/exceptions.html#NotImplementedError + # if a method is not supposed to be supported in a subclass we should set it to None + from_legacy_cache = None + + to_legacy_cache = None + + +class QuantizedCache(DynamicCache): + """ + A quantizer cache similar to what is described in the [KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache paper](https://arxiv.org/abs/2402.02750). + It allows the model to generate longer sequence length without allocating too much memory for Key and Value cache by applying quantization. + + The cache has two types of storage, one for original precision and one for the quantized cache. A `residual length` is set as a maximum capacity for the + original precision cache. When the length goes beyond maximum capacity, the original precision cache is discarded and moved into the quantized cache. The + quantization is done per-channel with a set `q_group_size` for both Keys and Values, in contrast to what was described in the paper. + + It stores Keys and Values a list of quantized tensors (tuples in case we need to store metadata), one for each layer. Additionally, it stores the Key and + Value in original precision states as a list of tensors, one for each layer. The size of each tensor + is `[batch_size, num_heads, seq_len - residual_length, head_dim]` + """ + + def __init__(self, cache_config: QuantizedCacheConfig) -> None: + super().__init__() + self._quantized_key_cache: List[torch.Tensor] = [] + self._quantized_value_cache: List[torch.Tensor] = [] + + self.nbits = cache_config.nbits + self.residual_length = cache_config.residual_length + self.q_group_size = cache_config.q_group_size + self.axis_key = cache_config.axis_key + self.axis_value = cache_config.axis_value + self.compute_dtype = cache_config.compute_dtype + self.device = cache_config.device + + super().__init__() + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Update the number of seen tokens + if layer_idx == 0: + self._seen_tokens += key_states.shape[-2] + + if len(self.key_cache) < layer_idx: + raise ValueError("QuantizedCache does not support model usage where layers are skipped. Use DynamicCache.") + elif len(self.key_cache) == layer_idx: + self._quantized_key_cache.append(self._quantize(key_states.contiguous(), axis=self.axis_key)) + self._quantized_value_cache.append(self._quantize(value_states.contiguous(), axis=self.axis_value)) + self.key_cache.append(torch.zeros(0, dtype=key_states.dtype, device=key_states.device)) + self.value_cache.append(torch.zeros(0, dtype=key_states.dtype, device=key_states.device)) + keys_to_return, values_to_return = key_states, value_states + else: + dequant_key = self._dequantize(self._quantized_key_cache[layer_idx]) + dequant_value = self._dequantize(self._quantized_value_cache[layer_idx]) + keys_to_return = [dequant_key, self.key_cache[layer_idx], key_states] + values_to_return = [dequant_value, self.value_cache[layer_idx], value_states] + + keys_to_return = torch.cat(keys_to_return, dim=-2) + values_to_return = torch.cat(values_to_return, dim=-2) + if ( + self.key_cache[layer_idx].dim() == 4 + and self.key_cache[layer_idx].shape[-2] + 1 >= self.residual_length + ): + self._quantized_key_cache[layer_idx] = self._quantize(keys_to_return.contiguous(), axis=self.axis_key) + self._quantized_value_cache[layer_idx] = self._quantize( + values_to_return.contiguous(), axis=self.axis_value + ) + self.key_cache[layer_idx] = torch.zeros(0, dtype=key_states.dtype, device=key_states.device) + self.value_cache[layer_idx] = torch.zeros(0, dtype=key_states.dtype, device=key_states.device) + else: + self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=-2) + self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=-2) + + return keys_to_return, values_to_return + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + if len(self.key_cache) <= layer_idx: + return 0 + # since we cannot get the seq_length of each layer directly and rely on `_seen_tokens` which is + # updated every "layer_idx" == 0, this is a hack to get the actual seq_length for the given layer_idx + # this part of code otherwise fails when used to verify attn_weight shape in some models + return self._seen_tokens if layer_idx == 0 else self._seen_tokens - 1 + + def _quantize(self, tensor, axis): + """Quantizes a key/value using a defined quantization method.""" + raise NotImplementedError("Make sure to implement `_quantize` in a subclass.") + + def _dequantize(self, q_tensor): + """Dequantizes back the tensor that was quantized by `self._quantize()`""" + raise NotImplementedError("Make sure to implement `_dequantize` in a subclass.") + + +class QuantoQuantizedCache(QuantizedCache): + """ + Quantized Cache class that uses `quanto` as a backend to perform quantization. Current implementation supports `int2` and `int4` dtypes only. + + Parameters: + cache_config (`QuantizedCacheConfig`): + A configuration containing all the arguments to be used by the quantizer, including axis, qtype and group size. + + Example: + + ```python + >>> # Run pip install quanto first if you don't have it yet + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, QuantoQuantizedCache, QuantizedCacheConfig + + >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + + >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> cache_config = QuantizedCacheConfig(nbits=4) + >>> past_key_values = QuantoQuantizedCache(cache_config=cache_config) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + QuantoQuantizedCache() + ``` + """ + + def __init__(self, cache_config: CacheConfig) -> None: + super().__init__(cache_config) + + if is_optimum_quanto_available(): + optimum_quanto_version = version.parse(importlib.metadata.version("optimum-quanto")) + if optimum_quanto_version <= version.parse("0.2.5"): + raise ImportError( + f"You need optimum-quanto package version to be greater or equal than 0.2.5 to use `QuantoQuantizedCache`. Detected version {optimum_quanto_version}." + ) + from optimum.quanto import MaxOptimizer, qint2, qint4 + + if self.nbits not in [2, 4]: + raise ValueError(f"`nbits` for `quanto` backend has to be one of [`2`, `4`] but got {self.nbits}") + + if self.axis_key not in [0, -1]: + raise ValueError(f"`axis_key` for `quanto` backend has to be one of [`0`, `-1`] but got {self.axis_key}") + + if self.axis_value not in [0, -1]: + raise ValueError( + f"`axis_value` for `quanto` backend has to be one of [`0`, `-1`] but got {self.axis_value}" + ) + + self.qtype = qint4 if self.nbits == 4 else qint2 + self.optimizer = MaxOptimizer() # hardcode as it's the only one for per-channel quantization + + def _quantize(self, tensor, axis): + # We have two different API since in optimum-quanto, we don't use AffineQuantizer anymore + if is_optimum_quanto_available(): + from optimum.quanto import quantize_weight + + scale, zeropoint = self.optimizer(tensor, self.qtype, axis, self.q_group_size) + qtensor = quantize_weight(tensor, self.qtype, axis, scale, zeropoint, self.q_group_size) + return qtensor + + def _dequantize(self, qtensor): + return qtensor.dequantize() + + +class HQQQuantizedCache(QuantizedCache): + """ + Quantized Cache class that uses `HQQ` as a backend to perform quantization. Current implementation supports `int2`, `int4`, `int8` dtypes. + + Parameters: + cache_config (`QuantizedCacheConfig`): + A configuration containing all the arguments to be used by the quantizer, including axis, qtype and group size. + + Example: + + ```python + >>> # Run pip install hqq first if you don't have it yet + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, HQQQuantizedCache, QuantizedCacheConfig + + >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + + >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> cache_config = QuantizedCacheConfig(nbits=4, axis_key=1, axis_value=1) + >>> past_key_values = HQQQuantizedCache(cache_config=cache_config) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + HQQQuantizedCache() + ``` + """ + + def __init__(self, cache_config: CacheConfig) -> None: + super().__init__(cache_config) + if self.nbits not in [1, 2, 3, 4, 8]: + raise ValueError( + f"`nbits` for `HQQ` backend has to be one of [`1`, `2`, `3`, `4`, `8`] but got {self.nbits}" + ) + + if self.axis_key not in [0, 1]: + raise ValueError(f"`axis_key` for `HQQ` backend has to be one of [`0`, `1`] but got {self.axis_key}") + + if self.axis_value not in [0, 1]: + raise ValueError(f"`axis_value` for `HQQ` backend has to be one of [`0`, `1`] but got {self.axis_value}") + + self.quantizer = HQQQuantizer + + def _quantize(self, tensor, axis): + qtensor, meta = self.quantizer.quantize( + tensor, + axis=axis, + device=self.device, + compute_dtype=self.compute_dtype, + nbits=self.nbits, + group_size=self.q_group_size, + ) + meta["compute_dtype"] = self.compute_dtype + self.quantizer.cuda(qtensor, meta=meta, device=self.device) # Move to device and cast to dtype + return qtensor, meta + + def _dequantize(self, qtensor): + quant_tensor, meta = qtensor + tensor = self.quantizer.dequantize(quant_tensor, meta) + return tensor + + +class SinkCache(Cache): + """ + A cache that as described in the [Attention Sinks paper](https://arxiv.org/abs/2309.17453). It allows the model to + generate beyond the length of its context window, without losing fluency in the conversation. As it discards past + tokens, the model will lose the ability to generate tokens that depend on the context that was discarded. + + It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is + `[batch_size, num_heads, seq_len, head_dim]`. + + Parameters: + window_length (`int`): + The length of the context window. + num_sink_tokens (`int`): + The number of sink tokens. See the original paper for more information. + + Example: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, SinkCache + + >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct") + + >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> past_key_values = SinkCache(window_length=256, num_sink_tokens=4) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + SinkCache() + ``` + """ + + is_sliding = True + + def __init__(self, window_length: int, num_sink_tokens: int) -> None: + super().__init__() + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + self.window_length = window_length + self.num_sink_tokens = num_sink_tokens + self.cos_sin_rerotation_cache = {} + self._cos_cache = None + self._sin_cache = None + self._seen_tokens = 0 # Used in `generate` to keep tally of how many tokens the cache has seen + + @staticmethod + def _rotate_half(x): + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + def _apply_key_rotary_pos_emb( + self, key_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor + ) -> torch.Tensor: + rotated_key_states = (key_states * cos) + (self._rotate_half(key_states) * sin) + return rotated_key_states + + def _get_rerotation_cos_sin( + self, key_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + if key_states.shape[-2] not in self.cos_sin_rerotation_cache: + # Upcast to float32 temporarily for better accuracy + cos = cos.to(torch.float32) + sin = sin.to(torch.float32) + + # Compute the cos and sin required for back- and forward-rotating to one position earlier in the sequence + original_cos = cos[self.num_sink_tokens + key_states.shape[-2] :] + shifted_cos = cos[self.num_sink_tokens : -key_states.shape[-2]] + original_sin = sin[self.num_sink_tokens + key_states.shape[-2] :] + shifted_sin = sin[self.num_sink_tokens : -key_states.shape[-2]] + rerotation_cos = original_cos * shifted_cos + original_sin * shifted_sin + rerotation_sin = -original_sin * shifted_cos + original_cos * shifted_sin + + self.cos_sin_rerotation_cache[key_states.shape[-2]] = ( + rerotation_cos.to(key_states.dtype).unsqueeze(0), + rerotation_sin.to(key_states.dtype).unsqueeze(0), + ) + return self.cos_sin_rerotation_cache[key_states.shape[-2]] + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # TODO: deprecate this function in favor of `cache_position` + # Workaround to make 'key_states.shape[-2] + past_key_value.get_seq_length(self.layer_idx)' <= window_length + if len(self.key_cache) <= layer_idx: + return 0 + return self.key_cache[layer_idx].shape[-2] + + def get_max_cache_shape(self) -> Optional[int]: + """Returns the maximum sequence length of the cache object, in case of SinkCache it is the window length.""" + return self.window_length + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. The following arguments can be used in `SinkCache`: `sin`, + `cos` and `partial_rotation_size`. These arguments are used with models using RoPE, to recompute the + rotation as the tokens are shifted. + + Return: + A tuple containing the updated key and value states. + """ + # Optional kwargs for `SinkCache` -- needed on models using RoPE. `partial_rotation_size` is used on models + # with partially rotated position embeddings, like Phi or Persimmon. + if cache_kwargs is None: + cache_kwargs = {} + sin = cache_kwargs.get("sin") + cos = cache_kwargs.get("cos") + partial_rotation_size = cache_kwargs.get("partial_rotation_size") + using_rope = cos is not None and sin is not None + + # Update the number of seen tokens + if layer_idx == 0: + self._seen_tokens += key_states.shape[-2] + + # Update the sin/cos cache, which holds sin/cos values for all possible positions + if using_rope and layer_idx == 0: + # BC: some models still pass `sin`/`cos` with 2 dims. In those models, they are the full sin/cos. Remove + # after all RoPE models have a llama-like cache utilization. + if cos.dim() == 2: + self._cos_cache = cos + self._sin_cache = sin + else: + if self._cos_cache is None: + self._cos_cache = cos[0, ...] + self._sin_cache = sin[0, ...] + elif self._cos_cache.shape[0] < self.window_length: + self._cos_cache = torch.cat([self._cos_cache, cos[0, ...]], dim=0) + self._sin_cache = torch.cat([self._sin_cache, sin[0, ...]], dim=0) + + # [bsz, num_heads, seq_len, head_dim] + if len(self.key_cache) <= layer_idx: + # Empty cache + self.key_cache.append(key_states) + self.value_cache.append(value_states) + + elif key_states.shape[-2] + self.get_seq_length(layer_idx) < self.window_length: + # Growing cache + self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=-2) + self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=-2) + + else: + # Shifting cache + keys_to_keep = self.key_cache[layer_idx][ + :, :, -self.window_length + self.num_sink_tokens + key_states.shape[-2] : + ] + + # On RoPE models, we need to recompute the Key rotation as the tokens are shifted + if using_rope: + rerotation_cos, rerotation_sin = self._get_rerotation_cos_sin( + key_states, self._cos_cache[: self.window_length], self._sin_cache[: self.window_length] + ) + if partial_rotation_size is not None: + keys_to_keep, keys_pass = ( + keys_to_keep[..., :partial_rotation_size], + keys_to_keep[..., partial_rotation_size:], + ) + keys_to_keep = self._apply_key_rotary_pos_emb(keys_to_keep, rerotation_cos, rerotation_sin) + if partial_rotation_size is not None: + keys_to_keep = torch.cat((keys_to_keep, keys_pass), dim=-1) + + # Concatenate sink tokens, shifted & rotated tokens (if needed), and new tokens + sink_keys = self.key_cache[layer_idx][:, :, : self.num_sink_tokens] + self.key_cache[layer_idx] = torch.cat([sink_keys, keys_to_keep, key_states], dim=-2) + + sink_values = self.value_cache[layer_idx][:, :, : self.num_sink_tokens] + values_to_keep = self.value_cache[layer_idx][ + :, :, -self.window_length + self.num_sink_tokens + value_states.shape[-2] : + ] + self.value_cache[layer_idx] = torch.cat([sink_values, values_to_keep, value_states], dim=-2) + + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + +class StaticCache(Cache): + """ + Static Cache class to be used with `torch.compile(model)` and `torch.export()`. + + Parameters: + config (`PretrainedConfig`): + The configuration file defining the shape-related attributes required to initialize the static cache. + max_batch_size (`int`): + The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a + smaller batch size is used. If you are manually setting the batch size, make sure to take into account the + number of beams if you are running beam search + max_cache_len (`int`, *optional*): + The maximum sequence length with which the model will be used. + device (`torch.device` or `str`, *optional*): + The device on which the cache should be initialized. If you're using more than 1 computation device, you + should pass the `layer_device_map` argument instead. + dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): + The default `dtype` to use when initializing the layer. + layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*): + Mapping between the layers and its device. This is required when you are manually initializing the cache + and the model is split between different gpus. You can know which layers mapped to which device by + checking the associated device_map: `model.hf_device_map`. + + + Example: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, StaticCache + + >>> model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-chat-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") + + >>> inputs = tokenizer(text="My name is Llama", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate + >>> max_generated_length = inputs.input_ids.shape[1] + 10 + >>> past_key_values = StaticCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + StaticCache() + ``` + """ + + is_compileable = True + + def __init__( + self, + config: PretrainedConfig, + max_batch_size: int, + max_cache_len: Optional[int] = None, + device: Union[torch.device, str, None] = None, + dtype: torch.dtype = torch.float32, + layer_device_map: Optional[Dict[int, Union[str, torch.device, int]]] = None, + ) -> None: + super().__init__() + self.max_batch_size = max_batch_size + self.max_cache_len = config.max_position_embeddings if max_cache_len is None else max_cache_len + + # Some model define a custom `head_dim` != config.hidden_size // config.num_attention_heads + self.head_dim = ( + config.head_dim if hasattr(config, "head_dim") else config.hidden_size // config.num_attention_heads + ) + + self._dtype = dtype + self.num_key_value_heads = ( + config.num_attention_heads + if getattr(config, "num_key_value_heads", None) is None + else config.num_key_value_heads + ) + + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + # Note: There will be significant perf decrease if switching to use 5D tensors instead. + cache_shape = (self.max_batch_size, self.num_key_value_heads, self.max_cache_len, self.head_dim) + device = torch.device(device) if device is not None else None + for idx in range(config.num_hidden_layers): + if layer_device_map is not None: + layer_device = layer_device_map[idx] + else: + layer_device = device + new_layer_key_cache = torch.zeros(cache_shape, dtype=self._dtype, device=layer_device) + new_layer_value_cache = torch.zeros(cache_shape, dtype=self._dtype, device=layer_device) + # Note: `mark_static_address` is used to tag the cache as a fixed data pointer, + # preventing compiled graph breaks when updating the cache. + torch._dynamo.mark_static_address(new_layer_key_cache) + torch._dynamo.mark_static_address(new_layer_value_cache) + self.key_cache.append(new_layer_key_cache) + self.value_cache.append(new_layer_value_cache) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + It is VERY important to index using a tensor, otherwise you introduce a copy to the device. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. The `StaticCache` needs the `cache_position` input + to know how where to write in the cache. + + Return: + A tuple containing the updated key and value states. + """ + if cache_kwargs is None: + cache_kwargs = {} + cache_position = cache_kwargs.get("cache_position") + k_out = self.key_cache[layer_idx] + v_out = self.value_cache[layer_idx] + key_states = key_states.to(k_out.dtype) + value_states = value_states.to(v_out.dtype) + + if cache_position is None: + k_out.copy_(key_states) + v_out.copy_(value_states) + else: + # Note: here we use `tensor.index_copy_(dim, index, tensor)` that is equivalent to + # `tensor[:, :, index] = tensor`, but the first one is compile-friendly and it does explicitly an in-place + # operation, that avoids copies and uses less memory. + try: + k_out.index_copy_(2, cache_position, key_states) + v_out.index_copy_(2, cache_position, value_states) + except NotImplementedError: + # The operator 'aten::index_copy.out' is not currently implemented for the MPS device. + k_out[:, :, cache_position] = key_states + v_out[:, :, cache_position] = value_states + + return k_out, v_out + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states that were seen by the model.""" + # Occupied cache == any slot in the 3rd dim (sequence length) holds a non-zero value. To save on compute, let's + # limit the check to the first batch member and head dimension. + # TODO: deprecate this function in favor of `cache_position` + return (self.key_cache[layer_idx][0, 0].any(dim=-1)).sum() + + def get_max_cache_shape(self) -> Optional[int]: + return self.max_cache_len + + def reset(self): + """Resets the cache values while preserving the objects""" + for layer_idx in range(len(self.key_cache)): + # In-place ops prevent breaking the static address + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() + + +class SlidingWindowCache(StaticCache): + """ + Sliding Window Cache class to be used with `torch.compile` for models like Mistral that support sliding window attention. + Every time when we try to update the cache, we compute the `indices` based on `cache_position >= self.config.sliding_window - 1`, + if true(which means the cache can not hold all the old key value states and new states together because of the sliding window constraint), + we need to do a cycle shift based on `indices` to replace the oldest states by the new key value states passed in. + + The `to_shift` is only true once we are above sliding_window. Thus with `sliding_window==64`: + + indices = (slicing + to_shift[-1].int()-1) % self.config.sliding_window + tensor([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 0]) + + We overwrite the cache using these, then we always write at cache_position (clamped to `sliding_window`) + + Parameters: + config (`PretrainedConfig`): + The configuration file defining the shape-related attributes required to initialize the static cache. + max_batch_size (`int`): + The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a + smaller batch size is used. + max_cache_len (`int`, *optional*): + The maximum sequence length with which the model will be used. + device (`torch.device` or `str`, *optional*): + The device on which the cache should be initialized. If you're using more than 1 computation device, you + should pass the `layer_device_map` argument instead. + dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): + The default `dtype` to use when initializing the layer. + layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*): + Mapping between the layers and its device. This is required when you are manually initializing the cache + and the model is split between different gpus. You can know which layers mapped to which device by + checking the associated device_map: `model.hf_device_map`. + + Example: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, SlidingWindowCache + + >>> model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3") + >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3") + + >>> inputs = tokenizer(text="My name is Mistral", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate + >>> max_generated_length = inputs.input_ids.shape[1] + 10 + >>> past_key_values = SlidingWindowCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + SlidingWindowCache() + ``` + """ + + is_sliding = True + is_compileable = True + + def __init__( + self, + config: PretrainedConfig, + max_batch_size: int, + max_cache_len: Optional[int] = None, + device: Union[torch.device, str, None] = None, + dtype: torch.dtype = torch.float32, + layer_device_map: Optional[Dict[int, Union[str, torch.device, int]]] = None, + ) -> None: + if not hasattr(config, "sliding_window") or config.sliding_window is None: + raise ValueError( + "Setting `cache_implementation` to 'sliding_window' requires the model config supporting " + "sliding window attention, please check if there is a `sliding_window` field in the model " + "config and it's not set to None." + ) + max_cache_len = min(config.sliding_window, max_cache_len) + super().__init__( + config=config, + max_batch_size=max_batch_size, + max_cache_len=max_cache_len, + device=device, + dtype=dtype, + layer_device_map=layer_device_map, + ) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if cache_kwargs is None: + cache_kwargs = {} + cache_position = cache_kwargs.get("cache_position") + k_out = self.key_cache[layer_idx] + v_out = self.value_cache[layer_idx] + key_states = key_states.to(k_out.dtype) + value_states = value_states.to(v_out.dtype) + + # assume this only happens in prefill phase when prompt length > sliding_window_size (= max_cache_len) + if cache_position.shape[0] > self.max_cache_len: + k_out = key_states[:, :, -self.max_cache_len :, :] + v_out = value_states[:, :, -self.max_cache_len :, :] + # Assumption: caches are all zeros at this point, `+=` is equivalent to `=` but compile-friendly + self.key_cache[layer_idx] += k_out + self.value_cache[layer_idx] += v_out + # we should return the whole states instead of k_out, v_out to take the whole prompt + # into consideration when building kv cache instead of just throwing away tokens outside of the window + return key_states, value_states + + slicing = torch.ones(self.max_cache_len, dtype=torch.long, device=value_states.device).cumsum(0) + cache_position = cache_position.clamp(0, self.max_cache_len - 1) + to_shift = cache_position >= self.max_cache_len - 1 + indices = (slicing + to_shift[-1].int() - 1) % self.max_cache_len + + k_out = k_out[:, :, indices] + v_out = v_out[:, :, indices] + + try: + k_out.index_copy_(2, cache_position, key_states) + v_out.index_copy_(2, cache_position, value_states) + except NotImplementedError: + # The operator 'aten::index_copy.out' is not currently implemented for the MPS device. + k_out[:, :, cache_position] = key_states + v_out[:, :, cache_position] = value_states + + # `_.zero()` followed by `+=` is equivalent `=`, but compile-friendly (without graph breaks due to assignment) + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() + + self.key_cache[layer_idx] += k_out + self.value_cache[layer_idx] += v_out + + return k_out, v_out + + def get_max_cache_shape(self) -> Optional[int]: + return self.max_cache_len + + def reset(self): + for layer_idx in range(len(self.key_cache)): + # In-place ops prevent breaking the static address + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() + + +class EncoderDecoderCache(Cache): + """ + Base, abstract class for all encoder-decoder caches. Can be used to hold combinations of self-attention and + cross-attention caches. + + Example: + + ```python + >>> from transformers import AutoProcessor, AutoModelForCausalLM, DynamicCache, EncoderDecoderCache + + >>> model = AutoModelForCausalLM.from_pretrained("openai/whisper-small") + >>> processor = AutoProcessor.from_pretrained("openai/whisper-small") + + >>> inputs = processor(audio=YOUR-AUDIO, return_tensors="pt") + + >>> # Prepare cache classes for encoder and decoder and pass it to model's forward + >>> self_attention_cache = DynamicCache() + >>> cross_attention_cache = DynamicCache() + >>> past_key_values = EncoderDecoderCache(self_attention_cache, cross_attention_cache) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + EncoderDecoderCache() + ``` + + """ + + def __init__(self, self_attention_cache: Cache, cross_attention_cache: Cache): + super().__init__() + self.self_attention_cache = self_attention_cache + self.cross_attention_cache = cross_attention_cache + self.is_compileable = getattr(self.self_attention_cache, "is_compileable", False) + + self.is_updated = {} + for layer_idx in range(len(cross_attention_cache.key_cache)): + self.is_updated[layer_idx] = bool(cross_attention_cache.get_seq_length(layer_idx) > 0) + + def __getitem__(self, layer_idx: int) -> List[Tuple[torch.Tensor]]: + """ + Support for backwards-compatible `past_key_value` indexing, e.g. `past_key_value[0][0].shape[2]` to get the + sequence length. + """ + if layer_idx < len(self): + return ( + self.self_attention_cache.key_cache[layer_idx], + self.self_attention_cache.value_cache[layer_idx], + self.cross_attention_cache.key_cache[layer_idx], + self.cross_attention_cache.value_cache[layer_idx], + ) + else: + raise KeyError(f"Cache only has {len(self)} layers, attempted to access layer with index {layer_idx}") + + def __len__(self): + """ + Support for backwards-compatible `past_key_value` length, e.g. `len(past_key_value)`. This value corresponds + to the number of layers in the model. + """ + return len(self.self_attention_cache) + + def to_legacy_cache(self) -> Tuple[Tuple[torch.Tensor], Tuple[torch.Tensor]]: + """Converts the `EncoderDecoderCache` instance into its equivalent in the legacy cache format.""" + legacy_cache = () + if len(self.cross_attention_cache) > 0: + for self_attn, cross_attn in zip( + self.self_attention_cache.to_legacy_cache(), self.cross_attention_cache.to_legacy_cache() + ): + legacy_cache += (self_attn + cross_attn,) + else: + legacy_cache = self.self_attention_cache.to_legacy_cache() + return legacy_cache + + @classmethod + def from_legacy_cache( + cls, past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None + ) -> "EncoderDecoderCache": + """Converts a cache in the legacy cache format into an equivalent `EncoderDecoderCache`.""" + cache = cls( + self_attention_cache=DynamicCache(), + cross_attention_cache=DynamicCache(), + ) + if past_key_values is not None: + for layer_idx in range(len(past_key_values)): + key_states, value_states = past_key_values[layer_idx][:2] + cache.self_attention_cache.update(key_states, value_states, layer_idx) + if len(past_key_values[layer_idx]) > 2: + key_states, value_states = past_key_values[layer_idx][2:] + cache.cross_attention_cache.update(key_states, value_states, layer_idx) + cache.is_updated[layer_idx] = True + return cache + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # check if empty list because in case of static cache it will be a tensors and we can't check `if not torch.Tensor` + return self.self_attention_cache.get_seq_length(layer_idx) + + def reset(self): + if hasattr(self.self_attention_cache, "reset"): + self.self_attention_cache.reset() + if hasattr(self.cross_attention_cache, "reset"): + self.cross_attention_cache.reset() + elif not hasattr(self.self_attention_cache, "reset") and not hasattr(self.cross_attention_cache, "reset"): + raise ValueError( + "Neither self nor cross-attention cache have valid `.reset()` methods. `.reset()` should " + "only be called on compatible cache classes, such as `StaticCache` or `SlidingWindowCache`. " + f"Got {self.self_attention_cache.__str__()} for the self attention cache and " + f"{self.cross_attention_cache.__str__()} for the cross attention cache." + ) + for layer_idx in self.is_updated: + self.is_updated[layer_idx] = False + + def reorder_cache(self, beam_idx: torch.LongTensor): + """Reorders the cache for beam search, given the selected beam indices.""" + self.self_attention_cache.reorder_cache(beam_idx) + self.cross_attention_cache.reorder_cache(beam_idx) + + def check_dynamic_cache(self, method: str): + if not ( + isinstance(self.self_attention_cache, DynamicCache) + and isinstance(self.cross_attention_cache, DynamicCache) + ): + raise ValueError( + f"`{method}` is only defined for dynamic cache, got {self.self_attention_cache.__str__()} for the self " + f"attention cache and {self.cross_attention_cache.__str__()} for the cross attention cache." + ) + + # TODO(gante, sanchit-gandhi): move following functionality into `.generate` + def crop(self, maximum_length: int): + """Crop the past key values up to a new `maximum_length` in terms of tokens. `maximum_length` can also be + negative to remove `maximum_length` tokens. This is used in assisted decoding and contrastive search.""" + self.check_dynamic_cache(self.crop.__name__) + self.self_attention_cache.crop(maximum_length) + + def batch_split(self, full_batch_size: int, split_size: int) -> "List[EncoderDecoderCache]": + """Split the current instance into a list of `DynamicCache` by the batch size. This will be used by + `_split_model_inputs()` in `generation.utils`""" + self.check_dynamic_cache(self.batch_split.__name__) + self_attention_cache = self.self_attention_cache.batch_split(full_batch_size, split_size) + cross_attention_cache = self.cross_attention_cache.batch_split(full_batch_size, split_size) + + out = [] + for self_attn, cross_attn in zip(self_attention_cache, cross_attention_cache): + out.append(EncoderDecoderCache(self_attn, cross_attn)) + return out + + @classmethod + def from_batch_splits(cls, splits: List["EncoderDecoderCache"]) -> "EncoderDecoderCache": + """This is the opposite of the above `batch_split()` method. This will be used by `stack_model_outputs` in + `generation.utils`""" + self_attention_cache = DynamicCache() + cross_attention_cache = DynamicCache() + for idx in range(len(splits[0])): + layer_keys = torch.cat([current.self_attention_cache.key_cache[idx] for current in splits], dim=0) + layer_values = torch.cat([current.self_attention_cache.value_cache[idx] for current in splits], dim=0) + self_attention_cache.update(layer_keys, layer_values, idx) + + layer_keys = torch.cat([current.cross_attention_cache.key_cache[idx] for current in splits], dim=0) + layer_values = torch.cat([current.cross_attention_cache.value_cache[idx] for current in splits], dim=0) + cross_attention_cache.update(layer_keys, layer_values, idx) + return cls(self_attention_cache, cross_attention_cache) + + def batch_repeat_interleave(self, repeats: int): + """Repeat the cache `repeats` times in the batch dimension. Used in contrastive search.""" + self.check_dynamic_cache(self.batch_repeat_interleave.__name__) + self.self_attention_cache.batch_repeat_interleave(repeats) + self.cross_attention_cache.batch_repeat_interleave(repeats) + + def batch_select_indices(self, indices: torch.Tensor): + """Only keep the `indices` in the batch dimension of the cache. Used in contrastive search.""" + self.check_dynamic_cache(self.batch_select_indices.__name__) + self.self_attention_cache.batch_select_indices(indices) + self.cross_attention_cache.batch_select_indices(indices) + + +class HybridCache(Cache): + """ + Hybrid Cache class to be used with `torch.compile` for Gemma2 models that alternate between a local sliding window attention + and global attention in every other layer. Under the hood, Hybrid Cache leverages ["SlidingWindowCache"] for sliding window attention + and ["StaticCache"] for global attention. For more information, see the documentation of each subcomponeent cache class. + + Parameters: + config (`PretrainedConfig): + The configuration file defining the shape-related attributes required to initialize the static cache. + max_batch_size (`int`): + The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a + smaller batch size is used. + max_cache_len (`int`, *optional*): + The maximum sequence length with which the model will be used. + device (`torch.device` or `str`, *optional*): + The device on which the cache should be initialized. If you're using more than 1 computation device, you + should pass the `layer_device_map` argument instead. + dtype (torch.dtype, *optional*, defaults to `torch.float32`): + The default `dtype` to use when initializing the layer. + layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*): + Mapping between the layers and its device. This is required when you are manually initializing the cache + and the model is split between different gpus. You can know which layers mapped to which device by + checking the associated device_map: `model.hf_device_map`. + + Example: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, HybridCache + + >>> model = AutoModelForCausalLM.from_pretrained("google/gemma-2-2b") + >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b") + + >>> inputs = tokenizer(text="My name is Gemma", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate + >>> max_generated_length = inputs.input_ids.shape[1] + 10 + >>> past_key_values = HybridCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + HybridCache() + ``` + """ + + # TODO (joao): dive deeper into gemma2 and paligemma -- there are reports of speed loss with compilation. Revert + # ALL changes from the PR that commented the line below when reactivating it. + # is_compileable = True + + def __init__( + self, + config: PretrainedConfig, + max_batch_size: int, + max_cache_len: Optional[int] = None, + device: Union[torch.device, str, None] = None, + dtype: torch.dtype = torch.float32, + layer_device_map: Optional[Dict[int, Union[str, torch.device, int]]] = None, + ) -> None: + super().__init__() + if not hasattr(config, "sliding_window") or config.sliding_window is None: + raise ValueError( + "Setting `cache_implementation` to 'sliding_window' requires the model config supporting " + "sliding window attention, please check if there is a `sliding_window` field in the model " + "config and it's not set to None." + ) + self.max_cache_len = max_cache_len + self.max_batch_size = max_batch_size + # Some model define a custom `head_dim` != config.hidden_size // config.num_attention_heads + self.head_dim = ( + config.head_dim if hasattr(config, "head_dim") else config.hidden_size // config.num_attention_heads + ) + + self._dtype = dtype + self.num_key_value_heads = ( + config.num_attention_heads if config.num_key_value_heads is None else config.num_key_value_heads + ) + + layer_switch = config.sliding_window_pattern if hasattr(config, "sliding_window_pattern") else 2 # 2 is for BC + self.is_sliding = torch.tensor( + [bool((i + 1) % layer_switch) for i in range(config.num_hidden_layers)], dtype=torch.bool + ) + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + global_cache_shape = (self.max_batch_size, self.num_key_value_heads, max_cache_len, self.head_dim) + sliding_cache_shape = ( + self.max_batch_size, + self.num_key_value_heads, + min(config.sliding_window, max_cache_len), + self.head_dim, + ) + device = torch.device(device) if device is not None and isinstance(device, str) else None + for i in range(config.num_hidden_layers): + if layer_device_map is not None: + layer_device = layer_device_map[i] + else: + layer_device = device + # Note: `mark_static_address` is used to tag the cache as an fixed data pointer, preventing cuda graph + # breaks when updating the cache. + cache_shape = global_cache_shape if not self.is_sliding[i] else sliding_cache_shape + new_layer_key_cache = torch.zeros(cache_shape, dtype=self._dtype, device=layer_device) + new_layer_value_cache = torch.zeros(cache_shape, dtype=self._dtype, device=layer_device) + torch._dynamo.mark_static_address(new_layer_key_cache) + torch._dynamo.mark_static_address(new_layer_value_cache) + self.key_cache.append(new_layer_key_cache) + self.value_cache.append(new_layer_value_cache) + + def _sliding_update(self, cache_position, layer_idx, key_states, value_states, k_out, v_out, max_cache_len): + if cache_position.shape[0] > max_cache_len: + k_out = key_states[:, :, -max_cache_len:, :] + v_out = value_states[:, :, -max_cache_len:, :] + # Assumption: caches are all zeros at this point, `+=` is equivalent to `=` but compile-friendly + self.key_cache[layer_idx] += k_out + self.value_cache[layer_idx] += v_out + # we should return the whole states instead of k_out, v_out to take the whole prompt + # into consideration when building kv cache instead of just throwing away tokens outside of the window + return key_states, value_states + + slicing = torch.ones(max_cache_len, dtype=torch.long, device=value_states.device).cumsum(0) + cache_position = cache_position.clamp(0, max_cache_len - 1) + to_shift = cache_position >= max_cache_len - 1 + indices = (slicing + to_shift[-1].int() - 1) % max_cache_len + k_out = k_out[:, :, indices] + v_out = v_out[:, :, indices] + + k_out[:, :, cache_position] = key_states + v_out[:, :, cache_position] = value_states + # `_.zero()` followed by `+=` is equivalent `=`, but compile-friendly (without graph breaks due to assignment) + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() + + self.key_cache[layer_idx] += k_out + self.value_cache[layer_idx] += v_out + return k_out, v_out + + def _static_update(self, cache_position, layer_idx, key_states, value_states, k_out, v_out, max_cache_len): + k_out[:, :, cache_position] = key_states + v_out[:, :, cache_position] = value_states + + self.key_cache[layer_idx] = k_out + self.value_cache[layer_idx] = v_out + return k_out, v_out + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if cache_kwargs is None: + cache_kwargs = {} + cache_position = cache_kwargs.get("cache_position") + sliding_window = cache_kwargs.get("sliding_window") + + # These two `if` blocks are only reached in multigpu and if `layer_device_map` is not passed. They are used + # when the cache is initialized in the forward pass (e.g. Gemma2) + if self.key_cache[layer_idx].device != key_states.device: + self.key_cache[layer_idx] = self.key_cache[layer_idx].to(key_states.device) + if self.value_cache[layer_idx].device != value_states.device: + self.value_cache[layer_idx] = self.value_cache[layer_idx].to(value_states.device) + + k_out = self.key_cache[layer_idx] + v_out = self.value_cache[layer_idx] + key_states = key_states.to(k_out.dtype) + value_states = value_states.to(v_out.dtype) + + if sliding_window: + update_fn = self._sliding_update + else: + update_fn = self._static_update + + return update_fn( + cache_position, + layer_idx, + key_states, + value_states, + k_out, + v_out, + k_out.shape[2], + ) + + def get_max_cache_shape(self) -> Optional[int]: + return self.max_cache_len + + def get_seq_length(self, layer_idx: Optional[int] = 0): + # Occupied cache == any slot in the 3rd dim (sequence length) holds a non-zero value. To save on compute, let's + # limit the check to the first batch member and head dimension. + # TODO: deprecate this function in favor of `cache_position` + if layer_idx != 0: + raise ValueError( + "`get_seq_length` on `HybridCache` may get inconsistent results depending on the layer index. " + "Using the `layer_idx` argument is not supported." + ) + return (self.key_cache[layer_idx][0, 0].any(dim=-1)).sum() + + def reset(self): + """Resets the cache values while preserving the objects""" + for layer_idx in range(len(self.key_cache)): + # In-place ops prevent breaking the static address + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() + + +class HybridChunkedCache(Cache): + """ + Hybrid Cache class to be used with `torch.compile` for Gemma2 models that alternate between a local sliding window attention + and global attention in every other layer. Under the hood, Hybrid Cache leverages ["SlidingWindowCache"] for sliding window attention + and ["StaticCache"] for global attention. For more information, see the documentation of each subcomponeent cache class. + + Parameters: + config (`PretrainedConfig): + The configuration file defining the shape-related attributes required to initialize the static cache. + max_batch_size (`int`): + The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a + smaller batch size is used. + max_cache_len (`int`, *optional*): + The maximum sequence length with which the model will be used. + device (`torch.device` or `str`, *optional*): + The device on which the cache should be initialized. If you're using more than 1 computation device, you + should pass the `layer_device_map` argument instead. + dtype (torch.dtype, *optional*, defaults to `torch.bfloat16`): + The default `dtype` to use when initializing the layer. + layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*): + Mapping between the layers and its device. This is required when you are manually initializing the cache + and the model is split between different gpus. You can know which layers mapped to which device by + checking the associated device_map: `model.hf_device_map`. + + Example: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, HybridCache + + >>> model = AutoModelForCausalLM.from_pretrained("google/gemma-2-2b") + >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b") + + >>> inputs = tokenizer(text="My name is Gemma", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate + >>> max_generated_length = inputs.input_ids.shape[1] + 10 + >>> past_key_values = HybridCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values # access cache filled with key/values from generation + HybridCache() + ``` + """ + + # TODO (joao): dive deeper into gemma2 and paligemma -- there are reports of speed loss with compilation. Revert + # ALL changes from the PR that commented the line below when reactivating it. + is_compileable = True + + def __init__( + self, + config: PretrainedConfig, + max_batch_size: int, + max_cache_len: Optional[int] = None, + device: Union[torch.device, str, None] = None, + dtype: torch.dtype = torch.bfloat16, + layer_device_map: Optional[Dict[int, Union[str, torch.device, int]]] = None, + ) -> None: + super().__init__() + if not hasattr(config, "sliding_window") or config.sliding_window is None: + self.sliding_window = getattr(config.get_text_config(), "attention_chunk_size", 8192) + else: + self.sliding_window = config.sliding_window + self.max_cache_len = max_cache_len + self.max_batch_size = max_batch_size + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self._dtype = dtype + + if hasattr(config.get_text_config(), "no_rope_layers"): + self.is_sliding = config.no_rope_layers + else: + layer_switch = getattr(config, "sliding_window_pattern", 2) + self.is_sliding = [bool((i + 1) % layer_switch) for i in range(config.num_hidden_layers)] + + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + self.cumulative_length = [0 for _ in range(config.num_hidden_layers)] + + def initialise_cache_layer(self, layer_idx, key_states): + if len(self.key_cache) > layer_idx: + return + + num_key_value_heads = key_states.shape[1] + device = key_states.device + global_cache_shape = (self.max_batch_size, num_key_value_heads, self.max_cache_len, self.head_dim) + sliding_cache_shape = ( + self.max_batch_size, + num_key_value_heads, + self.sliding_window, + self.head_dim, + ) + # Note: `mark_static_address` is used to tag the cache as an fixed data pointer, preventing cuda graph + # breaks when updating the cache. + cache_shape = sliding_cache_shape if self.is_sliding[layer_idx] else global_cache_shape + new_layer_key_cache = torch.zeros(cache_shape, dtype=self._dtype, device=device) + new_layer_value_cache = torch.zeros(cache_shape, dtype=self._dtype, device=device) + torch._dynamo.mark_static_address(new_layer_key_cache) + torch._dynamo.mark_static_address(new_layer_value_cache) + self.key_cache.append(new_layer_key_cache) + self.value_cache.append(new_layer_value_cache) + + def _sliding_update(self, cache_position, layer_idx, key_states, value_states, k_out, v_out, max_cache_len): + cumulative_length = self.cumulative_length[layer_idx] + # Update it now that we saved the value above + self.cumulative_length[layer_idx] += key_states.shape[-2] + is_full = cumulative_length >= max_cache_len + if is_full: + full_key_states = torch.cat((k_out[:, :, 1:, :], key_states), dim=-2) + full_value_states = torch.cat((v_out[:, :, 1:, :], value_states), dim=-2) + # Fast decoding path -> here as the effective size is still sliding window, it is extremely important + # to return `self.key_cache[layer_idx]` and `self.value_cache[layer_idx]`, as they have the fixed adress + # in memory (the values are the same as the full states, but not the address!!) + if key_states.shape[-2] == 1: + self.key_cache[layer_idx].copy_(full_key_states) + self.value_cache[layer_idx].copy_(full_value_states) + return self.key_cache[layer_idx], self.value_cache[layer_idx] + elif not is_full and cumulative_length + key_states.shape[2] > max_cache_len: + # Fast prefill path, no need to cat() in this case (which creates a copy even if cating from 0 dim) + if cumulative_length == 0: + full_key_states = key_states + full_value_states = value_states + else: + full_key_states = torch.cat((k_out[:, :, :cumulative_length, :], key_states), dim=-2) + full_value_states = torch.cat((v_out[:, :, :cumulative_length, :], value_states), dim=-2) + else: + self.key_cache[layer_idx].index_copy_(2, cache_position, key_states) + self.value_cache[layer_idx].index_copy_(2, cache_position, value_states) + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + self.key_cache[layer_idx].copy_(full_key_states[:, :, -max_cache_len:, :]) + self.value_cache[layer_idx].copy_(full_value_states[:, :, -max_cache_len:, :]) + # we should return the whole states instead of k_out, v_out to take the whole prompt + # into consideration when building kv cache instead of just throwing away tokens outside of the window + return full_key_states, full_value_states + + def _static_update(self, cache_position, layer_idx, key_states, value_states, k_out, v_out, max_cache_len): + k_out[:, :, cache_position] = key_states + v_out[:, :, cache_position] = value_states + + self.key_cache[layer_idx] = k_out + self.value_cache[layer_idx] = v_out + return k_out, v_out + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if cache_kwargs is None: + cache_kwargs = {} + cache_position = cache_kwargs.get("cache_position") + self.initialise_cache_layer(layer_idx, key_states) + + k_out = self.key_cache[layer_idx] + v_out = self.value_cache[layer_idx] + key_states = key_states.to(k_out.dtype) + value_states = value_states.to(v_out.dtype) + + if self.is_sliding[layer_idx]: + update_fn = self._sliding_update + else: + update_fn = self._static_update + + return update_fn( + cache_position, + layer_idx, + key_states, + value_states, + k_out, + v_out, + k_out.shape[2], + ) + + def get_max_cache_shape(self) -> Optional[int]: + return self.max_cache_len + + def get_seq_length(self, layer_idx: Optional[int] = 0): + # Occupied cache == any slot in the 3rd dim (sequence length) holds a non-zero value. To save on compute, let's + # limit the check to the first batch member and head dimension. + # TODO: deprecate this function in favor of `cache_position` + if layer_idx != 0: + raise ValueError( + "`get_seq_length` on `HybridCache` may get inconsistent results depending on the layer index. " + "Using the `layer_idx` argument is not supported." + ) + if len(self.key_cache) == 0: + return 0 + return (self.key_cache[layer_idx][0, 0].any(dim=-1)).sum() + + def reset(self): + """Resets the cache values while preserving the objects""" + for layer_idx in range(len(self.key_cache)): + # In-place ops prevent breaking the static address + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() + self.cumulative_length = [0 for _ in range(len(self.cumulative_length))] + + +class MambaCache: + """ + Cache for mamba model which does not have attention mechanism and key value states. + + Arguments: + config (`PretrainedConfig): + The configuration file defining the shape-related attributes required to initialize the static cache. + max_batch_size (`int`): + The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a smaller batch size is used. + dtype (`torch.dtype`, *optional*, defaults to `torch.float16`): + The default `dtype` to use when initializing the layer. + device (`torch.device` or `str`, *optional*): + The device on which the cache should be initialized. Should be the same as the layer. + + Example: + + ```python + >>> from transformers import AutoTokenizer, MambaForCausalLM, MambaCache + + >>> model = MambaForCausalLM.from_pretrained("state-spaces/mamba-130m-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("state-spaces/mamba-130m-hf") + + >>> inputs = tokenizer(text="My name is Mamba", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> past_key_values = MambaCache(config=model.config, max_batch_size=1, device=model.device, dtype=model.dtype) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> outputs.past_key_values + MambaCache() + ``` + """ + + is_compileable = True + + # TODO (joao): add layer_device_map arg and update code in `generate` accordingly + def __init__( + self, + config: PretrainedConfig, + max_batch_size: int, + dtype: torch.dtype = torch.float16, + device: Union[torch.device, str, None] = None, + ): + self.max_batch_size = max_batch_size + self._dtype = dtype + self.intermediate_size = config.intermediate_size + self.ssm_state_size = config.state_size + self.conv_kernel_size = config.conv_kernel + + self.conv_states: List[torch.Tensor] = [] + self.ssm_states: List[torch.Tensor] = [] + device = torch.device(device) if device is not None else None + for _ in range(config.num_hidden_layers): + conv_state: torch.Tensor = torch.zeros( + self.max_batch_size, + self.intermediate_size, + self.conv_kernel_size, + device=device, + dtype=self._dtype, + ) + ssm_state: torch.Tensor = torch.zeros( + self.max_batch_size, + self.intermediate_size, + self.ssm_state_size, + device=device, + dtype=self._dtype, + ) + + torch._dynamo.mark_static_address(conv_state) + torch._dynamo.mark_static_address(ssm_state) + self.conv_states.append(conv_state) + self.ssm_states.append(ssm_state) + + def update_conv_state( + self, layer_idx: int, new_conv_state: torch.Tensor, cache_position: torch.LongTensor + ) -> torch.Tensor: + # This `if` blocks is only reached in multigpu and if `layer_device_map` is not passed. It is used + # when the cache is initialized in the forward pass (e.g. Mamba) + if self.conv_states[layer_idx].device != new_conv_state.device: + self.conv_states[layer_idx] = self.conv_states[layer_idx].to(new_conv_state.device) + + conv_state = self.conv_states[layer_idx] + cache_position = cache_position.clamp(0, self.conv_kernel_size - 1) + + conv_state = conv_state.roll(shifts=-1, dims=-1) + conv_state[:, :, cache_position] = new_conv_state.to(device=conv_state.device, dtype=conv_state.dtype) + self.conv_states[layer_idx].zero_() + self.conv_states[layer_idx] += conv_state + return self.conv_states[layer_idx] + + def update_ssm_state(self, layer_idx: int, new_ssm_state: torch.Tensor): + self.ssm_states[layer_idx] = new_ssm_state.to(self.ssm_states[layer_idx].device) + return self.ssm_states[layer_idx] + + def reset(self): + for layer_idx in range(len(self.conv_states)): + # In-place ops prevent breaking the static address + self.conv_states[layer_idx].zero_() + self.ssm_states[layer_idx].zero_() + + +class OffloadedStaticCache(StaticCache): + """ + Static cache class to be used with `torch.compile(model)` that offloads to the CPU or + another device. + + Args: + config (`PretrainedConfig): + The configuration file defining the shape-related attributes required to initialize + the static cache. + max_batch_size (`int`): + The maximum batch size with which the model will be used. + max_cache_len (`int`): + The maximum sequence length with which the model will be used. + device (`Union[str, torch.device]`): + The device on which the cache should be initialized. If you're using more than 1 computation device, you + should pass the `layer_device_map` argument instead. + dtype (`torch.dtype`, *optional*): + The default `dtype` to use when initializing the cache. + offload_device (`Union[str, torch.device]`, *optional*, defaults to `cpu`): + The device to offload to. Defaults to CPU. + layer_device_map (`Dict[int, Union[str, torch.device, int]]`, *optional*): + Mapping between the layers and its device. This is required when you are manually initializing the cache + and the model is splitted between differents gpus. You can know which layers mapped to which device by + checking the associated device_map: `model.hf_device_map`. + + Example: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, OffloadedStaticCache + + >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") + + >>> inputs = tokenizer(text="My name is GPT2", return_tensors="pt") + + >>> # Prepare a cache class and pass it to model's forward + >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate + >>> max_generated_length = inputs.input_ids.shape[1] + 10 + >>> past_key_values = OffloadedStaticCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype) + >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) + >>> past_kv_length = outputs.past_key_values # access cache filled with key/values from generation + ``` + """ + + is_compileable = True + + def __init__( + self, + config: PretrainedConfig, + max_batch_size: int, + max_cache_len: Optional[int], + device: Union[str, torch.device], + dtype: Optional[torch.dtype] = None, + offload_device: Union[str, torch.device] = torch.device("cpu"), + layer_device_map: Optional[Dict[int, Union[str, torch.device, int]]] = None, + ) -> None: + super(Cache, self).__init__() + self.max_batch_size = max_batch_size + self.max_cache_len = config.max_position_embeddings if max_cache_len is None else max_cache_len + self.device = torch.device(device) if layer_device_map is None else torch.device(layer_device_map[0]) + self.offload_device = torch.device(offload_device) + self._dtype = dtype if dtype is not None else torch.float32 + + # Some model define a custom `head_dim` != config.hidden_size // config.num_attention_heads + head_dim = config.head_dim if hasattr(config, "head_dim") else config.hidden_size // config.num_attention_heads + + num_key_value_heads = ( + config.num_attention_heads + if getattr(config, "num_key_value_heads", None) is None + else config.num_key_value_heads + ) + + cache_shape = (max_batch_size, num_key_value_heads, self.max_cache_len, head_dim) + + # Create offloaded CPU tensors. + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + + for i in range(config.num_hidden_layers): + # First layer is always on-device. + device = self.device if i == 0 else self.offload_device + + key_cache, value_cache = self._create_key_value_cache_tensors(cache_shape, device) + + self.key_cache.append(key_cache) + self.value_cache.append(value_cache) + + # Create device tensors. + self._device_key_cache: List[torch.Tensor] = [] + self._device_value_cache: List[torch.Tensor] = [] + + for i in range(2): + key_cache, value_cache = self._create_key_value_cache_tensors(cache_shape, self.device) + + self._device_key_cache.append(key_cache) + self._device_value_cache.append(value_cache) + + # For backwards compatibility. + # TODO(gante): Remove this. + self._seen_tokens = 0 + + # Create new CUDA stream for parallel prefetching. + self._prefetch_stream = torch.cuda.Stream() if self.device.type == "cuda" else None + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + It is VERY important to index using a tensor, otherwise you introduce a copy to the device. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, *optional*): + Additional arguments for the cache subclass. The `OffloadedStaticCache` needs the + `cache_position` input to know how where to write in the cache. + + Return: + A tuple containing the updated key and value states. + """ + + if layer_idx == 0: + # Update seen tokens. + # TODO(gante): Remove this. + self._seen_tokens += key_states.shape[-2] + + # Always there. + k_out = self.key_cache[0] + v_out = self.value_cache[0] + else: + # Wait for prefetch stream. + if self._prefetch_stream is not None: + torch.cuda.default_stream(self.device).wait_stream(self._prefetch_stream) + + k_out = self._device_key_cache[layer_idx & 1] + v_out = self._device_value_cache[layer_idx & 1] + + self._prefetch_layer(layer_idx + 1) + + cache_position = cache_kwargs.get("cache_position") if cache_kwargs is not None else None + if cache_position is None: + k_out.copy_(key_states) + v_out.copy_(value_states) + + # Copy the values to the offloaded device as well. + if layer_idx == 0: + self.key_cache[layer_idx].copy_(key_states.to(self.offload_device)) + self.value_cache[layer_idx].copy_(value_states.to(self.offload_device)) + else: + # Note: here we use `tensor.index_copy_(dim, index, tensor)` that is equivalent to + # `tensor[:, :, index] = tensor`, but the first one is compile-friendly and it does + # explicitly an in-place operation, that avoids copies and uses less memory. + try: + k_out.index_copy_(2, cache_position, key_states) + v_out.index_copy_(2, cache_position, value_states) + except NotImplementedError: + # The operator 'aten::index_copy.out' is not currently implemented for the MPS + # device. + k_out[:, :, cache_position] = key_states + v_out[:, :, cache_position] = value_states + + # Copy the values to the offloaded device as well. + if layer_idx != 0: + cache_position = cache_position.to(self.offload_device) + key_states = key_states.to(self.offload_device) + value_states = value_states.to(self.offload_device) + + try: + self.key_cache[layer_idx].index_copy_(2, cache_position, key_states) + self.value_cache[layer_idx].index_copy_(2, cache_position, value_states) + except NotImplementedError: + # The operator 'aten::index_copy.out' is not currently implemented for the MPS + # device. + self.key_cache[layer_idx][:, :, cache_position] = key_states + self.value_cache[layer_idx][:, :, cache_position] = value_states + + return k_out, v_out + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states that were seen by the model.""" + + # TODO(gante): Remove this. + return self._seen_tokens + + def get_max_cache_shape(self) -> Optional[int]: + """Returns the maximum sequence length of the cached states.""" + + return self.max_cache_len + + def reset(self) -> None: + """Resets the cache values while preserving the objects.""" + + # For backwards compatibility. + # TODO(gante): Remove this. + self._seen_tokens = 0 + + # Zero out cache. + for layer_idx in range(len(self.key_cache)): + # In-place ops prevent breaking the static address. + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() + + @property + def seen_tokens(self) -> int: + # For backwards compatibility. + # TODO(gante): Remove this. + return self._seen_tokens + + def _create_key_value_cache_tensors( + self, shape: Tuple[int, ...], device: torch.device + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Creates K/V cache tensors on a device. Pins memory for CPU tensors. Marks them as static + addresses for non-CPU tensors. + + Args: + shape (`Tuple[int, ...]`): Shape. + device (`torch.device`): Device. + + Returns: + Key and value cache tensors as a tuple. + """ + + is_cpu_device = device == torch.device("cpu") + + key_cache = torch.zeros(shape, dtype=self._dtype, device=device, pin_memory=is_cpu_device) + value_cache = torch.zeros(shape, dtype=self._dtype, device=device, pin_memory=is_cpu_device) + + # Note: `mark_static_address` is used to tag the cache as a fixed data pointer, + # preventing compiled graph breaks when updating the cache. + torch._dynamo.mark_static_address(key_cache) + torch._dynamo.mark_static_address(value_cache) + + return key_cache, value_cache + + def _prefetch_layer(self, layer_idx: int) -> None: + """Prefetch a layer to the device. Needs to be called in order of layer indices.""" + + # Don't fetch layers that do not exist. + if layer_idx >= len(self.key_cache): + return + + # Alternate between two on-device caches. + if self._prefetch_stream is not None: + with torch.cuda.stream(self._prefetch_stream): + self._prefetch_layer_in_context(layer_idx) + else: + self._prefetch_layer_in_context(layer_idx) + + def _prefetch_layer_in_context(self, layer_idx: int) -> None: + """Performs the actual copy of the layer to device cache.""" + + self._device_key_cache[layer_idx & 1].copy_(self.key_cache[layer_idx], non_blocking=True) + self._device_value_cache[layer_idx & 1].copy_(self.value_cache[layer_idx], non_blocking=True) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/candidate_generator.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/candidate_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..fe57f532e68750af1e38bb0387dabf594b3d3b82 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/candidate_generator.py @@ -0,0 +1,1160 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import weakref +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple + +import numpy as np +import torch + +from ..utils import is_sklearn_available + + +if is_sklearn_available(): + from sklearn.metrics import roc_curve + +from ..cache_utils import DynamicCache +from ..pytorch_utils import isin_mps_friendly +from .logits_process import LogitsProcessorList, MinLengthLogitsProcessor, SuppressTokensLogitsProcessor + + +if TYPE_CHECKING: + from ..modeling_utils import PreTrainedModel + from ..tokenization_utils_base import PreTrainedTokenizerBase + from .configuration_utils import GenerationConfig + + +class CandidateGenerator: + """Abstract base class for all candidate generators that can be applied during assisted generation.""" + + def get_candidates(self, input_ids: torch.LongTensor) -> Tuple[torch.LongTensor, Optional[torch.FloatTensor]]: + """ + Fetches the candidates to be tried for the current input. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + + Return: + `torch.LongTensor` of shape `(batch_size, candidate_length)` containing the candidate sequences to be + assessed by the model and, optionally, a `torch.FloatTensor` of shape `(batch_size, candidate_length, + vocabulary_size)` containing the logits associated to each candidate. + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can call `get_candidates`." + ) + + def update_candidate_strategy(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, num_matches: int): + """ + Updates the candidate generation strategy based on the outcomes. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + scores (`torch.FloatTensor` of shape `(batch_size, candidate_length, config.vocab_size)`): + Prediction scores of a language modeling head. These can be logits for each vocabulary when not using + beam search or log softmax for each vocabulary token when using beam search + num_matches (`int`): + The number of matches between the candidate sequences and the model predictions. + """ + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can call " + "`update_candidate_strategy`." + ) + + +class AssistedCandidateGenerator(CandidateGenerator): + """ + `CandidateGenerator` class to be used for assisted generation and speculative decoding. This class generates + candidates through the use of a smaller model. Read the following blog post for more information: + https://huggingface.co/blog/assisted-generation + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + assistant_model (`PreTrainedModel`): + The model to be used for generating candidates. This model should be smaller than the main model. + generation_config (`~generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. + logits_processor (`LogitsProcessorList`): + An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`] + used to modify the prediction scores of the language modeling head applied at each generation step. + model_kwargs (`Dict`): + The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant + model as well. + inputs_tensor (`torch.Tensor`, *optional*): + The model input tensor. In encoder-decoder models, this is the encoder input. + """ + + def __init__( + self, + input_ids: torch.LongTensor, + assistant_model: "PreTrainedModel", + generation_config: "GenerationConfig", + model_kwargs: Dict, + inputs_tensor: Optional[torch.Tensor] = None, + logits_processor: "LogitsProcessorList" = None, + ): + # Make sure all data at the same device as assistant model + device = assistant_model.device + input_ids = input_ids.to(device) + if inputs_tensor is not None: + inputs_tensor = inputs_tensor.to(device) + + # Prepare the assistant and the starting number of candidate tokens + self.assistant_model = assistant_model + self.num_assistant_tokens = assistant_model.generation_config.num_assistant_tokens + self.assistant_confidence_threshold = assistant_model.generation_config.assistant_confidence_threshold + + # Set eos in assistant same as in target model + self.assistant_model.generation_config.eos_token_id = generation_config.eos_token_id + + # Prepare the kwargs for the assistant model + assistant_kwargs = {} + for key, value in model_kwargs.items(): # deepcopy crashes if we attempt to copy encoder outputs with grads + if key not in ("encoder_outputs", "past_key_values"): + assistant_kwargs[key] = ( + value.detach().to(device) if isinstance(value, torch.Tensor) else copy.deepcopy(value) + ) + + # Remove potential default "logits_to_keep" key + if "logits_to_keep" in assistant_kwargs.keys() and not assistant_model._supports_logits_to_keep(): + del assistant_kwargs["logits_to_keep"] + + # If the assistant is an encoder-decoder model, assume the encoder is different on the assistant. + if assistant_model.config.is_encoder_decoder: + inputs_tensor, model_input_name, assistant_kwargs = assistant_model._prepare_model_inputs( + inputs_tensor, assistant_model.generation_config.bos_token_id, assistant_kwargs + ) + assistant_kwargs = assistant_model._prepare_encoder_decoder_kwargs_for_generation( + inputs_tensor, assistant_kwargs, model_input_name, assistant_model.generation_config + ) + elif "encoder_outputs" in model_kwargs: + assistant_kwargs["encoder_outputs"] = model_kwargs["encoder_outputs"] + self.assistant_kwargs = assistant_kwargs + + # Prepare assistant model's keys of inputs + if assistant_model.config.is_encoder_decoder: + # both are encoder-decoder + self.input_ids_key = "decoder_input_ids" + elif "encoder_outputs" in assistant_kwargs: + # special case for encoder-decoder with decoder-only assistant (like DistilWhisper) + self.input_ids_key = "input_ids" + self.assistant_kwargs["attention_mask"] = self.assistant_kwargs.get( + "decoder_attention_mask", + torch.ones((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.long), + ) + else: + # both are decoder-only + self.input_ids_key = "input_ids" + + # Prepare generation-related options. + self.logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList() + self.generation_config = copy.deepcopy(generation_config) + + self.generation_config.return_dict_in_generate = True + self.generation_config.output_scores = True + self.generation_config.assistant_confidence_threshold = self.assistant_confidence_threshold + # this flag allow us set the confidence stopping criteria for assistant model generation. + self.generation_config.is_assistant = True + + # avoid unnecessary warnings that min_length is larger than max_new_tokens + # remove the `MinLengthLogitsProcessor` if exists (NOTE: no need to check for `MinNewTokensLogitsProcessor`) + self.main_model_min_length = self.generation_config.min_length + self.generation_config.min_length = 0 + self.generation_config.min_new_tokens = None + for processor in self.logits_processor: + if isinstance(processor, MinLengthLogitsProcessor): + raise ValueError( + "Passing `MinLengthLogitsProcessor` when using `assisted_generation is disabled. " + "Please pass in `min_length` into `.generate()` instead" + ) + + # We need to roll back the cache in assisted generation, only DynamicCache is supported + self.generation_config.cache_implementation = None + + if ( + is_sklearn_available() + and self.assistant_model.generation_config.assistant_confidence_threshold + and type(self) is AssistedCandidateGenerator + ): + self.probs = [] + self.matches = [] + + def get_candidates(self, input_ids: torch.LongTensor) -> Tuple[torch.LongTensor, Optional[torch.FloatTensor]]: + """ + Fetches the candidates to be tried for the current input. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + + Return: + `torch.LongTensor` of shape `(batch_size, candidate_length)` containing the candidate sequences to be + assessed by the model and a `torch.FloatTensor` of shape `(batch_size, candidate_length, + vocabulary_size)` containing the logits associated to each candidate. + """ + input_ids = input_ids.to(self.assistant_model.device) + # Calculate new tokens to generate + min_new_tokens, max_new_tokens = self._calculate_new_tokens(input_ids) + if max_new_tokens == 0: + return input_ids, None + # Update past key values and masks + self._update_past_and_masks(input_ids) + # Generate candidates + generation_args = self._prepare_generation_args(input_ids, min_new_tokens, max_new_tokens) + candidate_ids, candidate_logits = self._generate_candidates(generation_args) + return candidate_ids, candidate_logits + + def update_candidate_strategy(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, num_matches: int): + """ + Updates the candidate generation strategy based on the outcomes. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + scores (`torch.FloatTensor` of shape `(batch_size, candidate_length, config.vocab_size)`): + Prediction scores of a language modeling head. These can be logits for each vocabulary when not using + beam search or log softmax for each vocabulary token when using beam search + num_matches (`int`): + The number of matches between the candidate sequences and the model predictions. + """ + # Adjust the max number of assistant tokens to use in the next iteration. This is a simple heuristic, + # probably can be improved -- we want to balance the benefits of getting assistant tokens correct with the + # cost of forecasting incorrect assistant tokens. + if self.assistant_model.generation_config.num_assistant_tokens_schedule in { + "heuristic", + "heuristic_transient", + }: + # len(scores[0])-1 is the number of candidates according to the target tokenizer. + if num_matches == len(scores[0]) - 1: + self.num_assistant_tokens += 2.0 + else: + self.num_assistant_tokens = max(1.0, self.num_assistant_tokens - 1.0) + + # The assistant's confidence threshold is adjusted throughout the speculative iterations to reduce the number of unnecessary draft and target forward passes. The costs are estimated based on the ROC curve, which considers the probability of the draft token and its match with the target. A cost of 25% is assigned to false positives and 75% to false negatives. + # This adaptation is not compatible with UAG, as it relies on the number of matched tokens based on the draft vocabulary, which is unavailable in UAG. + if ( + is_sklearn_available() + and self.assistant_model.generation_config.assistant_confidence_threshold + and type(self) is AssistedCandidateGenerator + ): + # update self.matches + self.matches.extend([1] * num_matches) + if len(self.probs) > len(self.matches): + self.matches.append(0) + + # update self.probs + excess_length = len(self.probs) - len(self.matches) + if excess_length > 0: + del self.probs[-excess_length:] + + if ( + len(self.probs) > 5 and {0, 1}.issubset(self.matches) + ): # require at least 5 samples to calculate the ROC curve and at least one positive and one negative sample + fpr, tpr, thresholds = roc_curve(self.matches, self.probs) + fnr = 1 - tpr + + # Calculate the cost for each threshold + costs = fpr + 3 * fnr + + # Find the threshold that minimizes the cost + optimal_threshold_index = np.argmin(costs) + best_threshold = thresholds[optimal_threshold_index] + + self.assistant_model.generation_config.assistant_confidence_threshold = best_threshold + + def _calculate_new_tokens(self, input_ids: torch.LongTensor) -> Tuple[int, int]: + """Calculate the minimum and maximum number of new tokens to generate.""" + new_cur_len = input_ids.shape[-1] + max_new_tokens = min(int(self.num_assistant_tokens), self.generation_config.max_length - new_cur_len - 1) + min_new_tokens = max(min(max_new_tokens, self.main_model_min_length - new_cur_len), 0) + return min_new_tokens, max_new_tokens + + def _update_past_and_masks( + self, input_ids: torch.LongTensor, remove_from_pkv: int = 0, num_added_tokens: int = 1 + ) -> bool: + """Update past key values and attention masks for subsequent generation rounds.""" + has_past_key_values = self.assistant_kwargs.get("past_key_values", None) is not None + if has_past_key_values: + new_cache_size = input_ids.shape[-1] - 1 - remove_from_pkv + self.assistant_kwargs["past_key_values"] = _crop_past_key_values( + self.assistant_model, self.assistant_kwargs["past_key_values"], new_cache_size - num_added_tokens + ) + self.assistant_kwargs = _prepare_attention_mask( + self.assistant_kwargs, input_ids.shape[-1], self.assistant_model.config.is_encoder_decoder + ) + self.assistant_kwargs = _prepare_token_type_ids(self.assistant_kwargs, input_ids.shape[-1]) + + return has_past_key_values + + def _prepare_generation_args(self, input_ids: torch.LongTensor, min_new_tokens: int, max_new_tokens: int) -> Dict: + """Prepare arguments for the generation call.""" + return { + self.input_ids_key: input_ids, + "min_new_tokens": min_new_tokens, + "max_new_tokens": max_new_tokens, + "generation_config": self.generation_config, + "logits_processor": self.logits_processor, + } + + def _generate_candidates(self, generation_args: Dict) -> Tuple[torch.LongTensor, Optional[torch.FloatTensor]]: + """Generate candidate sequences using the assistant model.""" + assistant_output = self.assistant_model.generate(**generation_args, **self.assistant_kwargs) + self.assistant_kwargs["past_key_values"] = assistant_output.past_key_values + if ( + is_sklearn_available() + and self.assistant_model.generation_config.assistant_confidence_threshold + and type(self) is AssistedCandidateGenerator + ): + scores_tensor = torch.cat(assistant_output.scores, dim=0) + scores_softmax = torch.softmax(scores_tensor, dim=-1) + ids = assistant_output.sequences[-1, -len(assistant_output.scores) :] + p = scores_softmax[range(len(ids)), ids] + self.probs.extend(p.tolist()) + candidate_logits = torch.stack(assistant_output.scores, dim=1) + candidate_ids = assistant_output.sequences + return candidate_ids, candidate_logits + + +class AssistedCandidateGeneratorDifferentTokenizers(AssistedCandidateGenerator): + """ + `CandidateGenerator` class to be used for Universal Assisted Generation (UAD): assisted generation with different tokenizers + for the assistant and main models. This class generates candidates through the use of a smaller + model. + + The main model input tokens are re-encoded into assistant model tokens, then candidate tokens are generated in the assistant encoding, which are + in turn re-encoded into main model candidate tokens. Validation then proceeds as explained above. + The re-encoding steps involve decoding token ids into text and then encoding the text using a different tokenizer. + Since re-encoding the tokens may result in tokenization discrepancies, UAD finds the longest common subsequence between the source and target encodings, + to ensure the new tokens include the correct prompt suffix. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + assistant_model (`PreTrainedModel`): + The model to be used for generating candidates. This model should be smaller than the main model. + target_tokenizer (`PreTrainedTokenizerBase`): + The tokenizer used for the target model. + assistant_tokenizer (`PreTrainedTokenizerBase`): + The tokenizer used for the assistant model. + generation_config (`~generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. + logits_processor (`LogitsProcessorList`): + An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`] + used to modify the prediction scores of the language modeling head applied at each generation step. + model_kwargs (`Dict`): + The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant + model as well. + inputs_tensor (`torch.Tensor`, *optional*): + The model input tensor. In encoder-decoder models, this is the encoder input. + """ + + def __init__( + self, + input_ids: torch.LongTensor, + assistant_model: "PreTrainedModel", + target_tokenizer: "PreTrainedTokenizerBase", + assistant_tokenizer: "PreTrainedTokenizerBase", + generation_config: "GenerationConfig", + model_kwargs: Dict, + inputs_tensor: Optional[torch.Tensor] = None, + logits_processor: "LogitsProcessorList" = None, + ): + super().__init__(input_ids, assistant_model, generation_config, model_kwargs, inputs_tensor, logits_processor) + + self.target_tokenizer = target_tokenizer + self.assistant_tokenizer = assistant_tokenizer + self.prev_target_ids_len: Optional[int] = None + self.prev_assistant_ids = None + self.target_lookbehind = assistant_model.generation_config.target_lookbehind + self.assistant_lookbehind = assistant_model.generation_config.assistant_lookbehind + + @staticmethod + def _get_longest_diag_dict(input_matrix, nonzero_idx): + """ + Calculates the length of the longest diagonal sequence in a given matrix. + Args: + input_matrix (torch.Tensor): The input matrix. + nonzero_idx (torch.Tensor): The indices of the non-zero elements in the matrix. + Returns: + dict: A dictionary where the keys are the indices of the non-zero elements and the values are the lengths of the longest diagonal sequences starting from those indices. + """ + + visited = set() + diags = {} + for idx in nonzero_idx: + start_idx = torch.clone(idx) + tuple_start_idx = tuple(start_idx.tolist()) + + if tuple_start_idx in visited: + continue + + visited.add(tuple_start_idx) + cur_diag_len = 1 + start_idx += 1 + while start_idx[0] < input_matrix.shape[0] and start_idx[1] < input_matrix.shape[1]: + tuple_start_idx = tuple(start_idx.tolist()) + visited.add(tuple_start_idx) + + if input_matrix[start_idx[0], start_idx[1]] == 1: + cur_diag_len += 1 + start_idx += 1 + else: + break + + diags[idx] = cur_diag_len + return diags + + @staticmethod + def _get_longest_diag_index(input_matrix): + """ + Returns the start index and length of the longest diagonal in the given input. + Args: + input_matrix (numpy.ndarray): The input matrix. + Returns: + tuple: A tuple containing the start index and length of the longest diagonal. + """ + + diags = AssistedCandidateGeneratorDifferentTokenizers._get_longest_diag_dict( + input_matrix, input_matrix.nonzero() + ) + diags_values = list(diags.values()) + diags_keys = list(diags.keys()) + best_diag = np.argmax(diags_values) + diag_start_index = diags_keys[best_diag] + diag_start_length = diags_values[best_diag] + return diag_start_index, diag_start_length + + @staticmethod + def _get_tokens_diag(prompt, prompt_plus_new_tokens): + """ + Input: + prompt: 2D array of shape (batch_size, prompt_length), represents the original prompt tokens + prompt_plus_new_tokens: 2D array of shape (batch_size, prompt_length), represents the suffix of the original prompt, with additional new tokens. + Output: + discrepancy_length: int, represents the number of tokens that need to be replaced from prompt + new_tokens_only: 2D array of shape (batch_size, new_token_length), represents the new tokens that are not in prompt + discrepancy_only: 2D array of shape (batch_size, discrepancy_length), represents the new tokens that are in prompt but not in prompt_plus_new_tokens + """ + compare_mat = prompt_plus_new_tokens.T == prompt + if not torch.is_tensor(compare_mat): + compare_mat = torch.tensor(compare_mat) + + compare_mat_int = compare_mat.to(int) + + if not compare_mat_int.any().item(): + # empty intersection between prompt and prompt_plus_new_tokens + return None, None, None + + longest_location, longest_diag_length = AssistedCandidateGeneratorDifferentTokenizers._get_longest_diag_index( + compare_mat_int + ) + new_token_start_index = longest_location[0] + longest_diag_length + discrepancy_with_old = longest_location[1] + longest_diag_length + discrepancy_length = (prompt.shape[1] - discrepancy_with_old).item() + new_tokens_only = prompt_plus_new_tokens[:, new_token_start_index + discrepancy_length :] + discrepancy_only = prompt_plus_new_tokens[ + :, new_token_start_index : new_token_start_index + discrepancy_length + ] + return discrepancy_length, new_tokens_only, discrepancy_only + + def convert_source_tokens_to_target_tokens( + self, + input_ids, + source_tokenizer, + destination_tokenizer, + ): + """ + Convert token IDs from one tokenizer to another. + Args: + input_ids: The input token IDs. + source_tokenizer: The source tokenizer. + destination_tokenizer: The destination tokenizer. + Returns: + The converted token IDs. + """ + text = source_tokenizer.batch_decode(input_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True) + dest_ids = destination_tokenizer(text, add_special_tokens=True, return_tensors="pt")["input_ids"] + return dest_ids.to(input_ids.device) + + def get_candidates(self, input_ids: torch.LongTensor) -> Tuple[torch.LongTensor, Optional[torch.FloatTensor]]: + """ + Fetches the candidates to be tried for the current input. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + + Return: + `torch.LongTensor` of shape `(batch_size, candidate_length)` containing the candidate sequences to be + assessed by the model and a `torch.FloatTensor` of shape `(batch_size, candidate_length, + vocabulary_size)` containing the logits associated to each candidate. + """ + max_new_tokens = int(self.num_assistant_tokens) + if max_new_tokens == 0: + return input_ids, None + + input_ids = input_ids.to(self.assistant_model.device) + remove_from_pkv = 0 + + assistant_input_ids, remove_from_pkv = self._prepare_assistant_input_ids(input_ids) + self.prev_assistant_ids = assistant_input_ids + + min_new_tokens = max(min(max_new_tokens, self.main_model_min_length - assistant_input_ids.shape[-1]), 0) + + self._update_past_and_masks(assistant_input_ids, remove_from_pkv) + generation_args = self._prepare_generation_args(assistant_input_ids, min_new_tokens, max_new_tokens) + self.assistant_kwargs.pop("attention_mask", None) + + assistant_output = self.assistant_model.generate(**generation_args, **self.assistant_kwargs) + new_target_ids = self._process_assistant_outputs(input_ids, assistant_output.sequences, assistant_input_ids) + + # Update state + self.prev_target_ids_len = input_ids.shape[1] + self.assistant_kwargs["past_key_values"] = assistant_output.past_key_values + self.prev_assistant_ids = assistant_output.sequences + + if self.prev_target_ids_len >= new_target_ids.shape[1]: + return input_ids, None + + return new_target_ids, None + + def _prepare_assistant_input_ids(self, input_ids: torch.LongTensor) -> Tuple[torch.LongTensor, int]: + """Converts target input IDs to assistant input IDs, handling discrepancies.""" + convert_kwargs = { + "source_tokenizer": self.target_tokenizer, + "destination_tokenizer": self.assistant_tokenizer, + } + remove_from_pkv = 0 + + if self.prev_assistant_ids is not None and self.prev_target_ids_len > self.target_lookbehind: + # input_ids contains all target prompt input ids and some new target input ids + start_index_in_target_window = self.prev_target_ids_len - self.target_lookbehind + + new_assistant_ids = self.convert_source_tokens_to_target_tokens( + input_ids[:, start_index_in_target_window:], **convert_kwargs + ) + prompt_use_length = new_assistant_ids.shape[1] + prompt_use = self.prev_assistant_ids[:, -prompt_use_length:] + + discrepancy_length, new_tokens_only, discrepancy_only = self._get_tokens_diag( + prompt_use, new_assistant_ids + ) + assistant_input_ids = self.prev_assistant_ids + + if new_tokens_only is not None: + if discrepancy_length > 0 and discrepancy_only.shape[1] > 0: + if discrepancy_length == discrepancy_only.shape[1]: + assistant_input_ids[:, -discrepancy_length:] = discrepancy_only + + elif discrepancy_length > discrepancy_only.shape[1]: + discrepancy_length_diff = discrepancy_length - discrepancy_only.shape[1] + assistant_input_ids = assistant_input_ids[:, :-discrepancy_length_diff] + assistant_input_ids[:, -discrepancy_only.shape[1] :] = discrepancy_only + + remove_from_pkv = discrepancy_length + + if new_tokens_only.shape[1] > 0: + assistant_input_ids = torch.cat([assistant_input_ids, new_tokens_only], dim=-1) + else: + # edge case: in case of no intersection between prompt and new_assistant_ids + assistant_input_ids = torch.cat([assistant_input_ids, new_assistant_ids], dim=-1) + else: + assistant_input_ids = self.convert_source_tokens_to_target_tokens(input_ids, **convert_kwargs) + self.prev_target_ids_len = input_ids.shape[1] + + return assistant_input_ids, remove_from_pkv + + def _process_assistant_outputs( + self, input_ids: torch.LongTensor, assistant_sequences: torch.LongTensor, assistant_input_ids: torch.LongTensor + ) -> torch.LongTensor: + """Processes assistant outputs to obtain target input IDs.""" + num_prev_assistant = self.prev_assistant_ids.shape[1] + start_assistant_look_index = num_prev_assistant - self.assistant_lookbehind + + new_target_ids_from_window = self.convert_source_tokens_to_target_tokens( + assistant_sequences[:, start_assistant_look_index:], + source_tokenizer=self.assistant_tokenizer, + destination_tokenizer=self.target_tokenizer, + ) + target_prompt_use_length = new_target_ids_from_window.shape[1] + + target_prompt_use = input_ids[:, -target_prompt_use_length:] + + _, target_new_tokens_only, _ = self._get_tokens_diag(target_prompt_use, new_target_ids_from_window) + + new_target_ids = input_ids + + if target_new_tokens_only is not None: + if target_new_tokens_only.shape[1] > 0: + new_target_ids = torch.cat([new_target_ids, target_new_tokens_only], dim=-1) + else: + # edge case: in case of no intersection between prompt and new_target_ids + new_target_ids = torch.cat([new_target_ids, new_target_ids_from_window], dim=-1) + + if hasattr(self.generation_config, "max_length"): + new_target_ids = new_target_ids[:, : self.generation_config.max_length] + + return new_target_ids + + +class AssistantToTargetTranslator: + """ + Translates token ids and logits between assistant and target model vocabularies. This class is used to handle + vocabulary mismatches when using different tokenizers for the assistant and target models in speculative decoding, + as introduced in the paper "Lossless Speculative Decoding Algorithms for Heterogeneous Vocabularies" + (https://www.arxiv.org/abs/2502.05202). + It maintains mappings between the two vocabularies and handles token/logit conversion. + + Args: + target_tokenizer (`PreTrainedTokenizerBase`): + The tokenizer used by the target (main) model. + assistant_tokenizer (`PreTrainedTokenizerBase`): + The tokenizer used by the assistant model. + assistant_model_device (`str`, defaults to "cpu"): + The device where the assistant model is located. Used for placing tensors. + target_vocab_size (`int`, *optional*): + The size of the target model's vocabulary. If not provided, will be inferred from the target tokenizer. + """ + + FILTER_VALUE: float = -float("Inf") # The value used to filter out unmapped tokens in the logits. + SUPPRESS_TOKEN_ID: int = -1 # The ID used to mark suppressed tokens in the mapping. + + def __init__( + self, + target_tokenizer: "PreTrainedTokenizerBase", + assistant_tokenizer: "PreTrainedTokenizerBase", + target_vocab_size: int, # required since target_vocab_size can be different from the length of target_tokenizer.get_vocab() + assistant_model_device: str = "cpu", + ): + self._target_tokenizer: "PreTrainedTokenizerBase" = target_tokenizer + self._assistant_tokenizer: "PreTrainedTokenizerBase" = assistant_tokenizer + self._assistant_model_device: str = assistant_model_device + self.target_vocab_size: int = target_vocab_size + self._assistant_to_target_input_ids, self.target_to_assistant_input_ids = ( + self._get_assistant_to_target_input_ids() + ) + self._suppress_input_ids: list[int] = self._get_suppress_input_ids() + self.logits_processors: Optional[LogitsProcessorList] = None + if len(self._suppress_input_ids) > 0: + # len(self._suppress_input_ids) = 0 if the assistant vocab is a subset of the target vocab + self.logits_processors = LogitsProcessorList( + [SuppressTokensLogitsProcessor(self._get_suppress_input_ids(), self._assistant_model_device)] + ) + + def _get_assistant_to_target_input_ids(self): + target_vocab = self._target_tokenizer.get_vocab() + assistant_vocab = self._assistant_tokenizer.get_vocab() + + space_str = " " + target_space_ids = self._target_tokenizer(space_str, add_special_tokens=False)["input_ids"] + if len(target_space_ids) > 0: + target_space_sign = self._target_tokenizer.convert_ids_to_tokens(target_space_ids)[0][0] + + assistant_space_ids = self._assistant_tokenizer(space_str, add_special_tokens=False)["input_ids"] + if len(assistant_space_ids) > 0: + assistant_space_sign = self._assistant_tokenizer.convert_ids_to_tokens(assistant_space_ids)[0][0] + + if target_space_sign != assistant_space_sign: + # If the assistant tokenizer has a different space sign than the target tokenizer, + # we need to replace the assistant space sign with the target space sign in the assistant_vocab. + assistant_vocab = { + ( + tok.replace(assistant_space_sign, target_space_sign, 1) + if tok.startswith(assistant_space_sign) + else tok + ): idx + for tok, idx in assistant_vocab.items() + } + + max_assistant_index = max(assistant_vocab.values()) + assistant_to_target_input_ids = torch.full((max_assistant_index + 1,), self.SUPPRESS_TOKEN_ID, dtype=int) + target_to_assistant_input_ids: Dict[int, int] = {} + for tok, assistant_id in assistant_vocab.items(): + target_id = target_vocab.get(tok) + if target_id is not None: + assistant_to_target_input_ids[assistant_id] = target_id + target_to_assistant_input_ids[target_id] = assistant_id + return assistant_to_target_input_ids.to(self._assistant_model_device), target_to_assistant_input_ids + + def _get_suppress_input_ids(self) -> list[int]: + """ + Get the input ids that are in the assistant vocab but not in the target vocab. + """ + return torch.where(self._assistant_to_target_input_ids == self.SUPPRESS_TOKEN_ID)[0] + + def get_target_ids( + self, assistant_input_ids, target_input_ids, assistant_candidate_ids: torch.LongTensor + ) -> torch.LongTensor: + """ + Return the target candidate ids that correspond to the assistant candidate ids. + Note that we have already the target ids for the prompt and we only need to find the target ids for the new tokens. + Moreover, assistant ids of the original prompt does not necessarily appear in _assistant_to_target_input_ids. + """ + + num_new_tokens = len(assistant_candidate_ids[0]) - assistant_input_ids.shape[1] + if num_new_tokens == 0: + return target_input_ids + else: + transformed_slice = self._assistant_to_target_input_ids[assistant_candidate_ids[0, -num_new_tokens:]] + return torch.cat((target_input_ids, transformed_slice.unsqueeze(0)), dim=1) + + def get_target_logits(self, assistant_logits: torch.FloatTensor) -> torch.FloatTensor: + """ + Return the target logits that correspond to the assistant logits. + """ + + target_shape: tuple[int, ...] = (*assistant_logits.shape[:-1], self.target_vocab_size) + target_logits: torch.FloatTensor = torch.full( + target_shape, self.FILTER_VALUE, device=self._assistant_model_device + ) + # Mask for valid indices + assistant_indices_mask = self._assistant_to_target_input_ids != self.SUPPRESS_TOKEN_ID + # Exclude invalid indices + target_logits_supported_indices = self._assistant_to_target_input_ids[assistant_indices_mask] + valid_assistant_logits = assistant_logits[..., : self._assistant_to_target_input_ids.shape[0]] + + target_logits[..., target_logits_supported_indices] = valid_assistant_logits[..., assistant_indices_mask] + + return target_logits + + +class AssistantVocabTranslatorCache: + """ + Cache for `AssistantToTargetTranslator` instances. The instances are computed at + pre-processing time, and this cache allows us to avoid recomputing them. + """ + + _cache = weakref.WeakKeyDictionary() + + @classmethod + def get_translator( + cls, + target_tokenizer: "PreTrainedTokenizerBase", + assistant_tokenizer: "PreTrainedTokenizerBase", + target_vocab_size: int, + assistant_model_device: str = "cpu", + ) -> AssistantToTargetTranslator: + assistant_dict = cls._cache.get(target_tokenizer) + if assistant_dict is None: + assistant_dict = weakref.WeakKeyDictionary() + cls._cache[target_tokenizer] = assistant_dict + + mapping = assistant_dict.get(assistant_tokenizer) + if mapping is None: + mapping = AssistantToTargetTranslator( + target_tokenizer, assistant_tokenizer, target_vocab_size, assistant_model_device + ) + assistant_dict[assistant_tokenizer] = mapping + + return mapping + + @classmethod + def cleanup(cls): + """ + Clean up dead references in the cache. + This removes entries where either the target_tokenizer or assistant_tokenizer + has been garbage collected. + """ + # Remove entries from the outer cache where the target_tokenizer is no longer alive + dead_keys = [key for key in cls._cache if key is None] + for key in dead_keys: + del cls._cache[key] + + # For each assistant_dict, remove entries where assistant_tokenizer is no longer alive + for assistant_dict in cls._cache.values(): + dead_keys = [key for key in assistant_dict if key is None] + for key in dead_keys: + del assistant_dict[key] + + +class UniversalSpeculativeDecodingGenerator(AssistedCandidateGeneratorDifferentTokenizers): + """ + `CandidateGenerator` class to be used for Universal Speculative Decoding (USD): speculative decoding with different tokenizers + for the assistant and main models. This class generates candidates through the use of a smaller model. + """ + + def __init__( + self, + input_ids: torch.LongTensor, + assistant_model: "PreTrainedModel", + target_tokenizer: "PreTrainedTokenizerBase", + assistant_tokenizer: "PreTrainedTokenizerBase", + generation_config: "GenerationConfig", + model_kwargs: Dict, + atm_translator: AssistantToTargetTranslator, + inputs_tensor: Optional[torch.Tensor] = None, + logits_processor: "LogitsProcessorList" = None, + ): + # Initialize translator before parent class + self._atm_translator = atm_translator + super().__init__( + input_ids, + assistant_model, + target_tokenizer, + assistant_tokenizer, + generation_config, + model_kwargs, + inputs_tensor, + logits_processor, + ) + # Track sequence lengths and previous assistant IDs + self._target_seq_len_with_candidates: int = 0 + self._prev_assistant_ids: Optional[torch.LongTensor] = None + + def get_candidates(self, input_ids: torch.LongTensor) -> Tuple[torch.LongTensor, Optional[torch.FloatTensor]]: + """ + Simplified version of get_candidates that uses the translator cache for token conversion. + """ + target_input_ids = input_ids.to(self.assistant_model.device) + assistant_input_ids, num_added_tokens = self._prepare_assistant_input_ids(target_input_ids) + min_new_tokens, max_new_tokens = self._calculate_new_tokens(target_input_ids) + + if max_new_tokens == 0: + return input_ids, None + + self._update_past_and_masks(assistant_input_ids, num_added_tokens=num_added_tokens) + generation_args = self._prepare_generation_args(assistant_input_ids, min_new_tokens, max_new_tokens) + + # Ensure scores are returned + generation_args["generation_config"].output_scores = True + generation_args["generation_config"].return_dict_in_generate = True + + # Generate and process outputs using translator + if self._atm_translator.logits_processors is not None: + generation_args["logits_processor"] = self._atm_translator.logits_processors + self._prev_assistant_ids, assistant_candidate_logits = self._generate_candidates(generation_args) + + # Use translator to convert tokens and logits + target_candidate_ids = self._atm_translator.get_target_ids( + assistant_input_ids, target_input_ids, self._prev_assistant_ids + ) + self._target_seq_len_with_candidates = target_candidate_ids.shape[-1] + target_candidate_logits = self._atm_translator.get_target_logits(assistant_candidate_logits) + + return target_candidate_ids, target_candidate_logits + + def _update_past_and_masks(self, assistant_input_ids: torch.LongTensor, num_added_tokens: int = 1) -> bool: + if self._prev_assistant_ids is None: + # Prepare attention mask for the first generation. + # For subsequent generations, the attention mask is updated in super()_update_past_and_masks. + self.assistant_kwargs = _prepare_attention_mask( + self.assistant_kwargs, assistant_input_ids.shape[-1], self.assistant_model.config.is_encoder_decoder + ) + return super()._update_past_and_masks(assistant_input_ids, num_added_tokens=num_added_tokens) + + def _prepare_assistant_input_ids(self, target_input_ids: torch.LongTensor) -> torch.LongTensor: + """ + Simplified token conversion that only processes new tokens. + """ + # Calculate new tokens since last call + target_seq_len = target_input_ids.shape[-1] + if self._target_seq_len_with_candidates == 0: + new_token_count = target_seq_len + else: + new_token_count = 1 + target_new_ids = target_input_ids[:, -new_token_count:] + + # Convert the new tokens + assistant_new_ids = None + if self._target_seq_len_with_candidates > 0: + # we have only one new token and we can directly convert it + assistant_new_ids = self._atm_translator.target_to_assistant_input_ids.get(target_new_ids[0].item()) + if assistant_new_ids is None: + target_new_text = self.target_tokenizer.batch_decode( + target_new_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True + ) + assistant_new_ids = self.assistant_tokenizer( + target_new_text, add_special_tokens=False, return_tensors="pt" + )["input_ids"].to(self.assistant_model.device) + else: + assistant_new_ids = torch.tensor([[assistant_new_ids]], device=self.assistant_model.device) + + # Update or initialize assistant IDs + if self._prev_assistant_ids is None: + assistant_input_ids = assistant_new_ids + else: + tokens_to_remove = self._target_seq_len_with_candidates + 1 - target_seq_len + # If the number of new tokens is greater than zero, truncate the previous assistant IDs + if tokens_to_remove > 0: + self._prev_assistant_ids = self._prev_assistant_ids[:, :-tokens_to_remove] + assistant_input_ids = torch.cat([self._prev_assistant_ids, assistant_new_ids], dim=-1) + assistant_input_ids = assistant_input_ids.to(dtype=torch.long) + + return assistant_input_ids, len(assistant_new_ids[0]) + + +class PromptLookupCandidateGenerator(CandidateGenerator): + """ + `CandidateGenerator` class to be used for prompt lookup generation. This class generates candidates by looking up + likely continuations in the provided prompt (input_ids) itself. + Read the following blog post for more information: https://github.com/apoorvumang/prompt-lookup-decoding + + Args: + max_matching_ngram_size (`int`): + The maximum ngram size to be considered for matching in the prompt + num_output_tokens (`int`): + The number of tokens to be output as candidate tokens. + max_length (`int`): + The number of total maximum tokens that can be generated. For decoder-only models that includes the prompt length. + Defaults to 20, which is the max length used as default in generation config. + """ + + def __init__( + self, + eos_token_id: Optional[torch.Tensor] = None, + num_output_tokens: int = 10, + max_matching_ngram_size: Optional[int] = None, + max_length: int = 20, + ): + self.num_output_tokens = num_output_tokens + self.max_matching_ngram_size = max_matching_ngram_size if max_matching_ngram_size else 2 + self.max_length = max_length + self.eos_token_id = eos_token_id + + if self.max_matching_ngram_size <= 0 or self.num_output_tokens <= 0: + raise ValueError("Invalid max_matching_ngram_size or num_output_tokens") + + def get_candidates(self, input_ids: torch.LongTensor) -> Tuple[torch.LongTensor, Optional[torch.FloatTensor]]: + """ + Fetches the candidates to be tried for the current input. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + + Return: + `torch.LongTensor` of shape `(num_candidates, candidate_length)`: The candidate sequences to be tried. + """ + input_length = input_ids.size(1) + + # Don't generate more than `max_length - 1` candidates since the target model generates one extra token. + if self.max_length == input_length + 1: + return input_ids, None + + chosen_ids = None + match_found = False + for ngram_size in range(min(self.max_matching_ngram_size, input_length - 1), 0, -1): + # Create sliding windows of size ngram_size + windows = input_ids.unfold(dimension=1, size=ngram_size, step=1) + + # Convert ngram to a tensor for comparison + ngram_tensor = input_ids[0, -ngram_size:] + + # Find where the windows match the ngram + matches = (windows == ngram_tensor).all(dim=2) + + # Get the indices of matches + match_indices = matches.nonzero(as_tuple=True)[1] + + # Iterate through match indices to find a valid continuation + for idx in match_indices: + start_idx = idx + ngram_size + end_idx = start_idx + self.num_output_tokens + end_idx = min(end_idx, input_length, self.max_length) + + if start_idx < end_idx: + chosen_ids = input_ids[0, start_idx:end_idx] + match_found = True + + # remove remaining candidate ids if an "eos" token is found, otherwise the target model may + # accept eos and the rest as valid, thus not stopping generation after "eos" + # NOTE: below code is written based on the fact that assisted decoding supports only bs=1 + mask = isin_mps_friendly(chosen_ids, self.eos_token_id) + match_indices_eos = torch.nonzero(mask) + if match_indices_eos.numel() > 0: + first_eos_index = match_indices_eos[0].item() + chosen_ids = chosen_ids[:first_eos_index] + break + if match_found: + break + + if chosen_ids is None or len(chosen_ids) == 0: + # In case we didn't find a match return the input sequence unchanged, reverts back to autoregressive decoding + return input_ids, None + + # Now need extend input_ids with chosen_ids + chosen_ids = chosen_ids.unsqueeze(0) + candidate_input_ids = torch.cat((input_ids, chosen_ids), dim=1) + # assisted_generation expects logits as well, but we don't have those here, so returning None + return candidate_input_ids, None + + def update_candidate_strategy(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, num_matches: int): + """ + Updates the candidate generation strategy based on the outcomes. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + scores (`torch.FloatTensor` of shape `(batch_size, candidate_length, config.vocab_size)`): + Prediction scores of a language modeling head. These can be logits for each vocabulary when not using + beam search or log softmax for each vocabulary token when using beam search + num_matches (`int`): + The number of matches between the candidate sequences and the model predictions. + """ + # Currently does nothing + return + + +class EarlyExitCandidateGenerator(AssistedCandidateGenerator): + """ + `CandidateGenerator` class to be used for assisted generation and speculative decoding. This class generates + candidates through the use of **the model itself**, exiting early. Can only be used with models that support early + exit, e.g., `facebook/layerskip-llama3.2-1B`. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + assistant_model (`PreTrainedModel`): + The original model. This model must support early exit (i.e. is trained to compute logits in earlier + layers). + generation_config (`~generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. + logits_processor (`LogitsProcessorList`): + An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`] + used to modify the prediction scores of the language modeling head applied at each generation step. + model_kwargs (`Dict`): + The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant + model as well. + inputs_tensor (`torch.Tensor`, *optional*): + The model input tensor. In encoder-decoder models, this is the encoder input. + """ + + def __init__( + self, + input_ids: torch.LongTensor, + assistant_model: "PreTrainedModel", + generation_config: "GenerationConfig", + model_kwargs: Dict, + inputs_tensor: Optional[torch.Tensor] = None, + logits_processor: "LogitsProcessorList" = None, + ): + super().__init__( + input_ids=input_ids, + assistant_model=assistant_model, + generation_config=generation_config, + model_kwargs=model_kwargs, + inputs_tensor=inputs_tensor, + logits_processor=logits_processor, + ) + # We have to move early exit out of the generation config, otherwise the assistant will also call `generate` + # with early exit + self.assistant_early_exit = self.generation_config.assistant_early_exit + self.generation_config.assistant_early_exit = None + + def get_candidates(self, input_ids: torch.LongTensor) -> Tuple[torch.LongTensor, Optional[torch.FloatTensor]]: + # Temporarily sets the number of hidden layers to the early exit value + base_model = getattr(self.assistant_model, self.assistant_model.base_model_prefix) + original_num_hidden_layers = base_model.config.num_hidden_layers + base_model.config.num_hidden_layers = self.assistant_early_exit + candidate_ids, candidate_logits = super().get_candidates(input_ids) + base_model.config.num_hidden_layers = original_num_hidden_layers + return candidate_ids, candidate_logits + + +def _crop_past_key_values(model, past_key_values, max_length): + """Crops the past key values up to a certain maximum length.""" + new_past = [] + if model.config.is_encoder_decoder: + for idx in range(len(past_key_values)): + new_past.append( + ( + past_key_values[idx][0][:, :, :max_length, :], + past_key_values[idx][1][:, :, :max_length, :], + past_key_values[idx][2], + past_key_values[idx][3], + ) + ) + past_key_values = tuple(new_past) + # gptbigcode is special and stores kv in shape (batch_size, seq_len, dim), if it's a multi_query model + elif "gptbigcode" in model.__class__.__name__.lower() or ( + model.config.architectures is not None and "gptbigcode" in model.config.architectures[0].lower() + ): + if model.config.multi_query: + for idx in range(len(past_key_values)): + past_key_values[idx] = past_key_values[idx][:, :max_length, :] + else: + for idx in range(len(past_key_values)): + past_key_values[idx] = past_key_values[idx][:, :, :max_length, :] + elif isinstance(past_key_values, DynamicCache): + past_key_values.crop(max_length) + elif past_key_values is not None: + for idx in range(len(past_key_values)): + if past_key_values[idx] != ([], []): + new_past.append( + ( + past_key_values[idx][0][:, :, :max_length, :], + past_key_values[idx][1][:, :, :max_length, :], + ) + ) + else: + new_past.append((past_key_values[idx][0], past_key_values[idx][1])) + past_key_values = tuple(new_past) + return past_key_values + + +def _prepare_attention_mask(model_kwargs: Dict[str, Any], new_length: int, is_encoder_decoder: bool) -> Dict[str, Any]: + """Expands or crops the model's mask for decoding purposes, to the defined length""" + + mask_key = "decoder_attention_mask" if is_encoder_decoder else "attention_mask" + if mask_key not in model_kwargs: + return model_kwargs + + mask = model_kwargs[mask_key] + mask_length_diff = new_length - mask.shape[1] + + if mask_length_diff < 0: + model_kwargs[mask_key] = mask[:, :mask_length_diff] + elif mask_length_diff > 0: + model_kwargs[mask_key] = torch.cat([mask, mask.new_ones((mask.shape[0], mask_length_diff))], dim=-1) + + # Handle cross attention models + if "cross_attention_mask" in model_kwargs: + # Mllama case + cross_mask = model_kwargs["cross_attention_mask"] + if mask_length_diff < 0: + model_kwargs["cross_attention_mask"] = cross_mask[:, :mask_length_diff] + elif mask_length_diff > 0: + new_mask = cross_mask[:, -1:, :, :].repeat(1, mask_length_diff, 1, 1) + model_kwargs["cross_attention_mask"] = torch.cat([cross_mask, new_mask], dim=1) + elif "image_attention_mask" in model_kwargs: + # IDEFICS case + cross_mask = model_kwargs["image_attention_mask"] + if mask_length_diff < 0: + model_kwargs["image_attention_mask"] = cross_mask[:, :mask_length_diff] + elif mask_length_diff > 0: + new_mask = cross_mask[:, -1:, :].repeat(1, mask_length_diff, 1) + model_kwargs["image_attention_mask"] = torch.cat([cross_mask, new_mask], dim=1) + + return model_kwargs + + +def _prepare_token_type_ids(model_kwargs: Dict[str, Any], new_length: int) -> Dict[str, Any]: + """Expands or crops the model's token_type_ids for decoding purposes, to the defined length""" + if "token_type_ids" not in model_kwargs or model_kwargs["token_type_ids"] is None: + return model_kwargs + + token_type_ids = model_kwargs["token_type_ids"] + final_token_type = token_type_ids[:, -1].unsqueeze(-1) + type_length_diff = new_length - token_type_ids.shape[1] + + if type_length_diff < 0: + token_type_ids = token_type_ids[:, :type_length_diff] + elif type_length_diff > 0: + token_type_copies = final_token_type.repeat(1, type_length_diff) + model_kwargs["token_type_ids"] = torch.cat([model_kwargs["token_type_ids"], token_type_copies], dim=-1) + return model_kwargs diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/chat_template_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/chat_template_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f96b5d8ddecab445a6706d9562de3e1c6af43d0c --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/chat_template_utils.py @@ -0,0 +1,435 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import json +import re +import types +from contextlib import contextmanager +from datetime import datetime +from functools import lru_cache +from typing import Any, Callable, Optional, Union, get_args, get_origin, get_type_hints + +from packaging import version + +from .import_utils import is_jinja_available, is_torch_available, is_vision_available + + +if is_jinja_available(): + import jinja2 + from jinja2.ext import Extension + from jinja2.sandbox import ImmutableSandboxedEnvironment +else: + jinja2 = None + +if is_vision_available(): + from PIL.Image import Image + +if is_torch_available(): + from torch import Tensor + + +BASIC_TYPES = (int, float, str, bool, Any, type(None), ...) +# Extracts the initial segment of the docstring, containing the function description +description_re = re.compile(r"^(.*?)[\n\s]*(Args:|Returns:|Raises:|\Z)", re.DOTALL) +# Extracts the Args: block from the docstring +args_re = re.compile(r"\n\s*Args:\n\s*(.*?)[\n\s]*(Returns:|Raises:|\Z)", re.DOTALL) +# Splits the Args: block into individual arguments +args_split_re = re.compile( + r""" +(?:^|\n) # Match the start of the args block, or a newline +\s*(\w+):\s* # Capture the argument name and strip spacing +(.*?)\s* # Capture the argument description, which can span multiple lines, and strip trailing spacing +(?=\n\s*\w+:|\Z) # Stop when you hit the next argument or the end of the block +""", + re.DOTALL | re.VERBOSE, +) +# Extracts the Returns: block from the docstring, if present. Note that most chat templates ignore the return type/doc! +returns_re = re.compile(r"\n\s*Returns:\n\s*(.*?)[\n\s]*(Raises:|\Z)", re.DOTALL) + + +class TypeHintParsingException(Exception): + """Exception raised for errors in parsing type hints to generate JSON schemas""" + + pass + + +class DocstringParsingException(Exception): + """Exception raised for errors in parsing docstrings to generate JSON schemas""" + + pass + + +def _get_json_schema_type(param_type: str) -> dict[str, str]: + type_mapping = { + int: {"type": "integer"}, + float: {"type": "number"}, + str: {"type": "string"}, + bool: {"type": "boolean"}, + type(None): {"type": "null"}, + Any: {}, + } + if is_vision_available(): + type_mapping[Image] = {"type": "image"} + if is_torch_available(): + type_mapping[Tensor] = {"type": "audio"} + return type_mapping.get(param_type, {"type": "object"}) + + +def _parse_type_hint(hint: str) -> dict: + origin = get_origin(hint) + args = get_args(hint) + + if origin is None: + try: + return _get_json_schema_type(hint) + except KeyError: + raise TypeHintParsingException( + "Couldn't parse this type hint, likely due to a custom class or object: ", hint + ) + + elif origin is Union or (hasattr(types, "UnionType") and origin is types.UnionType): + # Recurse into each of the subtypes in the Union, except None, which is handled separately at the end + subtypes = [_parse_type_hint(t) for t in args if t is not type(None)] + if len(subtypes) == 1: + # A single non-null type can be expressed directly + return_dict = subtypes[0] + elif all(isinstance(subtype["type"], str) for subtype in subtypes): + # A union of basic types can be expressed as a list in the schema + return_dict = {"type": sorted([subtype["type"] for subtype in subtypes])} + else: + # A union of more complex types requires "anyOf" + return_dict = {"anyOf": subtypes} + if type(None) in args: + return_dict["nullable"] = True + return return_dict + + elif origin is list: + if not args: + return {"type": "array"} + else: + # Lists can only have a single type argument, so recurse into it + return {"type": "array", "items": _parse_type_hint(args[0])} + + elif origin is tuple: + if not args: + return {"type": "array"} + if len(args) == 1: + raise TypeHintParsingException( + f"The type hint {str(hint).replace('typing.', '')} is a Tuple with a single element, which " + "we do not automatically convert to JSON schema as it is rarely necessary. If this input can contain " + "more than one element, we recommend " + "using a List[] type instead, or if it really is a single element, remove the Tuple[] wrapper and just " + "pass the element directly." + ) + if ... in args: + raise TypeHintParsingException( + "Conversion of '...' is not supported in Tuple type hints. " + "Use List[] types for variable-length" + " inputs instead." + ) + return {"type": "array", "prefixItems": [_parse_type_hint(t) for t in args]} + + elif origin is dict: + # The JSON equivalent to a dict is 'object', which mandates that all keys are strings + # However, we can specify the type of the dict values with "additionalProperties" + out = {"type": "object"} + if len(args) == 2: + out["additionalProperties"] = _parse_type_hint(args[1]) + return out + + raise TypeHintParsingException("Couldn't parse this type hint, likely due to a custom class or object: ", hint) + + +def _convert_type_hints_to_json_schema(func: Callable) -> dict: + type_hints = get_type_hints(func) + signature = inspect.signature(func) + required = [] + for param_name, param in signature.parameters.items(): + if param.annotation == inspect.Parameter.empty: + raise TypeHintParsingException(f"Argument {param.name} is missing a type hint in function {func.__name__}") + if param.default == inspect.Parameter.empty: + required.append(param_name) + + properties = {} + for param_name, param_type in type_hints.items(): + properties[param_name] = _parse_type_hint(param_type) + + schema = {"type": "object", "properties": properties} + if required: + schema["required"] = required + + return schema + + +def parse_google_format_docstring(docstring: str) -> tuple[Optional[str], Optional[dict], Optional[str]]: + """ + Parses a Google-style docstring to extract the function description, + argument descriptions, and return description. + + Args: + docstring (str): The docstring to parse. + + Returns: + The function description, arguments, and return description. + """ + + # Extract the sections + description_match = description_re.search(docstring) + args_match = args_re.search(docstring) + returns_match = returns_re.search(docstring) + + # Clean and store the sections + description = description_match.group(1).strip() if description_match else None + docstring_args = args_match.group(1).strip() if args_match else None + returns = returns_match.group(1).strip() if returns_match else None + + # Parsing the arguments into a dictionary + if docstring_args is not None: + docstring_args = "\n".join([line for line in docstring_args.split("\n") if line.strip()]) # Remove blank lines + matches = args_split_re.findall(docstring_args) + args_dict = {match[0]: re.sub(r"\s*\n+\s*", " ", match[1].strip()) for match in matches} + else: + args_dict = {} + + return description, args_dict, returns + + +def get_json_schema(func: Callable) -> dict: + """ + This function generates a JSON schema for a given function, based on its docstring and type hints. This is + mostly used for passing lists of tools to a chat template. The JSON schema contains the name and description of + the function, as well as the names, types and descriptions for each of its arguments. `get_json_schema()` requires + that the function has a docstring, and that each argument has a description in the docstring, in the standard + Google docstring format shown below. It also requires that all the function arguments have a valid Python type hint. + + Although it is not required, a `Returns` block can also be added, which will be included in the schema. This is + optional because most chat templates ignore the return value of the function. + + Args: + func: The function to generate a JSON schema for. + + Returns: + A dictionary containing the JSON schema for the function. + + Examples: + ```python + >>> def multiply(x: float, y: float): + >>> ''' + >>> A function that multiplies two numbers + >>> + >>> Args: + >>> x: The first number to multiply + >>> y: The second number to multiply + >>> ''' + >>> return x * y + >>> + >>> print(get_json_schema(multiply)) + { + "name": "multiply", + "description": "A function that multiplies two numbers", + "parameters": { + "type": "object", + "properties": { + "x": {"type": "number", "description": "The first number to multiply"}, + "y": {"type": "number", "description": "The second number to multiply"} + }, + "required": ["x", "y"] + } + } + ``` + + The general use for these schemas is that they are used to generate tool descriptions for chat templates that + support them, like so: + + ```python + >>> from transformers import AutoTokenizer + >>> from transformers.utils import get_json_schema + >>> + >>> def multiply(x: float, y: float): + >>> ''' + >>> A function that multiplies two numbers + >>> + >>> Args: + >>> x: The first number to multiply + >>> y: The second number to multiply + >>> return x * y + >>> ''' + >>> + >>> multiply_schema = get_json_schema(multiply) + >>> tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01") + >>> messages = [{"role": "user", "content": "What is 179 x 4571?"}] + >>> formatted_chat = tokenizer.apply_chat_template( + >>> messages, + >>> tools=[multiply_schema], + >>> chat_template="tool_use", + >>> return_dict=True, + >>> return_tensors="pt", + >>> add_generation_prompt=True + >>> ) + >>> # The formatted chat can now be passed to model.generate() + ``` + + Each argument description can also have an optional `(choices: ...)` block at the end, such as + `(choices: ["tea", "coffee"])`, which will be parsed into an `enum` field in the schema. Note that this will + only be parsed correctly if it is at the end of the line: + + ```python + >>> def drink_beverage(beverage: str): + >>> ''' + >>> A function that drinks a beverage + >>> + >>> Args: + >>> beverage: The beverage to drink (choices: ["tea", "coffee"]) + >>> ''' + >>> pass + >>> + >>> print(get_json_schema(drink_beverage)) + ``` + { + 'name': 'drink_beverage', + 'description': 'A function that drinks a beverage', + 'parameters': { + 'type': 'object', + 'properties': { + 'beverage': { + 'type': 'string', + 'enum': ['tea', 'coffee'], + 'description': 'The beverage to drink' + } + }, + 'required': ['beverage'] + } + } + """ + doc = inspect.getdoc(func) + if not doc: + raise DocstringParsingException( + f"Cannot generate JSON schema for {func.__name__} because it has no docstring!" + ) + doc = doc.strip() + main_doc, param_descriptions, return_doc = parse_google_format_docstring(doc) + + json_schema = _convert_type_hints_to_json_schema(func) + if (return_dict := json_schema["properties"].pop("return", None)) is not None: + if return_doc is not None: # We allow a missing return docstring since most templates ignore it + return_dict["description"] = return_doc + for arg, schema in json_schema["properties"].items(): + if arg not in param_descriptions: + raise DocstringParsingException( + f"Cannot generate JSON schema for {func.__name__} because the docstring has no description for the argument '{arg}'" + ) + desc = param_descriptions[arg] + enum_choices = re.search(r"\(choices:\s*(.*?)\)\s*$", desc, flags=re.IGNORECASE) + if enum_choices: + schema["enum"] = [c.strip() for c in json.loads(enum_choices.group(1))] + desc = enum_choices.string[: enum_choices.start()].strip() + schema["description"] = desc + + output = {"name": func.__name__, "description": main_doc, "parameters": json_schema} + if return_dict is not None: + output["return"] = return_dict + return {"type": "function", "function": output} + + +def _render_with_assistant_indices( + compiled_template, messages, tools, documents, add_generation_prompt, **template_kwargs +): + rendered_blocks = [] + generation_indices = [] + with compiled_template.environment.activate_tracker(rendered_blocks, generation_indices): + for block in compiled_template.generate( + messages=messages, + tools=tools, + documents=documents, + add_generation_prompt=add_generation_prompt, + **template_kwargs, + ): + rendered_blocks.append(block) + rendered_chat = "".join(rendered_blocks) + return rendered_chat, generation_indices + + +@lru_cache +def _compile_jinja_template(chat_template): + if not is_jinja_available(): + raise ImportError( + "apply_chat_template requires jinja2 to be installed. Please install it using `pip install jinja2`." + ) + + class AssistantTracker(Extension): + # This extension is used to track the indices of assistant-generated tokens in the rendered chat + tags = {"generation"} + + def __init__(self, environment: ImmutableSandboxedEnvironment): + # The class is only initiated by jinja. + super().__init__(environment) + environment.extend(activate_tracker=self.activate_tracker) + self._rendered_blocks = None + self._generation_indices = None + + def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.CallBlock: + lineno = next(parser.stream).lineno + body = parser.parse_statements(["name:endgeneration"], drop_needle=True) + return jinja2.nodes.CallBlock(self.call_method("_generation_support"), [], [], body).set_lineno(lineno) + + @jinja2.pass_eval_context + def _generation_support(self, context: jinja2.nodes.EvalContext, caller: jinja2.runtime.Macro) -> str: + rv = caller() + if self.is_active(): + # Only track generation indices if the tracker is active + start_index = len("".join(self._rendered_blocks)) + end_index = start_index + len(rv) + self._generation_indices.append((start_index, end_index)) + return rv + + def is_active(self) -> bool: + return self._rendered_blocks or self._generation_indices + + @contextmanager + def activate_tracker(self, rendered_blocks: list[int], generation_indices: list[int]): + try: + if self.is_active(): + raise ValueError("AssistantTracker should not be reused before closed") + self._rendered_blocks = rendered_blocks + self._generation_indices = generation_indices + + yield + finally: + self._rendered_blocks = None + self._generation_indices = None + + if version.parse(jinja2.__version__) < version.parse("3.1.0"): + raise ImportError( + f"apply_chat_template requires jinja2>=3.1.0 to be installed. Your version is {jinja2.__version__}." + ) + + def raise_exception(message): + raise jinja2.exceptions.TemplateError(message) + + def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False): + # We override the built-in tojson filter because Jinja's default filter escapes HTML characters + # We also expose some options like custom indents and separators + return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys) + + def strftime_now(format): + return datetime.now().strftime(format) + + jinja_env = ImmutableSandboxedEnvironment( + trim_blocks=True, lstrip_blocks=True, extensions=[AssistantTracker, jinja2.ext.loopcontrols] + ) + jinja_env.filters["tojson"] = tojson + jinja_env.globals["raise_exception"] = raise_exception + jinja_env.globals["strftime_now"] = strftime_now + return jinja_env.from_string(chat_template) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/config.json b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1eba51348345490eeb487aad7cad675503969c1b --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoModel": "modeling_qwen3.Qwen3ForCausalLM", + "AutoModelForCausalLM": "modeling_qwen3_hyp.Qwen3HypForCausalLM" + }, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1280, + "initializer_range": 0.02, + "intermediate_size": 3840, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "lorentz_k0": 1.0, + "max_position_embeddings": 40960, + "max_window_layers": 20, + "model_type": "qwen3", + "num_attention_heads": 20, + "num_hidden_layers": 20, + "num_key_value_heads": 10, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/config.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/config.py new file mode 100644 index 0000000000000000000000000000000000000000..460ee9329977bdb2aadc54aae72e0cd93792aab0 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/config.py @@ -0,0 +1,741 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import dataclasses +import warnings +from abc import ABC, abstractmethod +from collections import OrderedDict +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Union + +import numpy as np +from packaging import version + +from ..utils import TensorType, is_torch_available, is_vision_available, logging +from .utils import ParameterFormat, compute_effective_axis_dimension, compute_serialized_parameters_size + + +if TYPE_CHECKING: + from ..configuration_utils import PretrainedConfig + from ..feature_extraction_utils import FeatureExtractionMixin + from ..image_processing_utils import ImageProcessingMixin + from ..tokenization_utils_base import PreTrainedTokenizerBase + + +if is_vision_available(): + from PIL import Image + +logger = logging.get_logger(__name__) + + +DEFAULT_ONNX_OPSET = 11 + +# 2 Gb +EXTERNAL_DATA_FORMAT_SIZE_LIMIT = 2 * 1024 * 1024 * 1024 + + +@dataclasses.dataclass +class PatchingSpec: + """ + Data class that holds patching specifications. + + Args: + o: Module / object where the op to patch is located + name: Name of the op to monkey patch + custom_op: Custom op that patches the original op + orig_op: Original op that is being patched + op_wrapper: Wrapper (optional) that wraps both the original and custom ops. + It is useful for ops that are class or static methods for instance. + """ + + o: Any + name: str + custom_op: Callable + orig_op: Optional[Callable] = None + op_wrapper: Optional[Callable] = None + + +class OnnxConfig(ABC): + """ + Base class for ONNX exportable model describing metadata on how to export the model through the ONNX format. + """ + + default_fixed_batch = 2 + default_fixed_sequence = 8 + default_fixed_num_choices = 4 + torch_onnx_minimum_version = version.parse("1.8") + _tasks_to_common_outputs = { + "causal-lm": OrderedDict({"logits": {0: "batch", 1: "sequence"}}), + "default": OrderedDict({"last_hidden_state": {0: "batch", 1: "sequence"}}), + "image-classification": OrderedDict({"logits": {0: "batch", 1: "sequence"}}), + "image-segmentation": OrderedDict( + { + "logits": {0: "batch", 1: "sequence"}, + "pred_boxes": {0: "batch", 1: "sequence"}, + "pred_masks": {0: "batch", 1: "sequence"}, + } + ), + "masked-im": OrderedDict({"logits": {0: "batch", 1: "sequence"}}), + "masked-lm": OrderedDict({"logits": {0: "batch", 1: "sequence"}}), + "multiple-choice": OrderedDict({"logits": {0: "batch"}}), + "object-detection": OrderedDict( + { + "logits": {0: "batch", 1: "sequence"}, + "pred_boxes": {0: "batch", 1: "sequence"}, + } + ), + "question-answering": OrderedDict( + { + "start_logits": {0: "batch", 1: "sequence"}, + "end_logits": {0: "batch", 1: "sequence"}, + } + ), + "semantic-segmentation": OrderedDict({"logits": {0: "batch", 1: "num_labels", 2: "height", 3: "width"}}), + "seq2seq-lm": OrderedDict({"logits": {0: "batch", 1: "decoder_sequence"}}), + "sequence-classification": OrderedDict({"logits": {0: "batch"}}), + "token-classification": OrderedDict({"logits": {0: "batch", 1: "sequence"}}), + "vision2seq-lm": OrderedDict({"logits": {0: "batch", 1: "sequence"}}), + "speech2seq-lm": OrderedDict({"logits": {0: "batch", 1: "sequence"}}), + } + + def __init__(self, config: "PretrainedConfig", task: str = "default", patching_specs: List[PatchingSpec] = None): + self._config = config + + if task not in self._tasks_to_common_outputs: + raise ValueError( + f"{task} is not a supported task, supported tasks: {self._tasks_to_common_outputs.keys()}" + ) + self.task = task + + self._patching_specs = [] + for spec in patching_specs if patching_specs is not None else []: + final_spec = spec + if spec.orig_op is None: + final_spec = dataclasses.replace(spec, orig_op=getattr(spec.o, spec.name)) + self._patching_specs.append(final_spec) + + @classmethod + def from_model_config(cls, config: "PretrainedConfig", task: str = "default") -> "OnnxConfig": + """ + Instantiate a OnnxConfig for a specific model + + Args: + config: The model's configuration to use when exporting to ONNX + + Returns: + OnnxConfig for this model + """ + return cls(config, task=task) + + @property + @abstractmethod + def inputs(self) -> Mapping[str, Mapping[int, str]]: + """ + Mapping containing the axis definition of the input tensors to provide to the model + + Returns: + For each input: its name associated to the axes symbolic name and the axis position within the tensor + """ + raise NotImplementedError() + + @property + def outputs(self) -> Mapping[str, Mapping[int, str]]: + """ + Mapping containing the axis definition of the output tensors to provide to the model + + Returns: + For each output: its name associated to the axes symbolic name and the axis position within the tensor + """ + common_outputs = self._tasks_to_common_outputs[self.task] + return copy.deepcopy(common_outputs) + + @property + def values_override(self) -> Optional[Mapping[str, Any]]: + """ + Dictionary of keys to override in the model's config before exporting + + Returns: + Dictionary with the keys (and their corresponding values) to override + """ + if hasattr(self._config, "use_cache"): + return {"use_cache": False} + + return None + + @property + def default_batch_size(self) -> int: + """ + The default batch size to use if no other indication + + Returns: + Integer > 0 + """ + # Using 2 avoid ONNX making assumption about single sample batch + return OnnxConfig.default_fixed_batch + + @property + def default_sequence_length(self) -> int: + """ + The default sequence length to use if no other indication + + Returns: + Integer > 0 + """ + return OnnxConfig.default_fixed_sequence + + @property + def default_num_choices(self) -> int: + """ + The default number of choices to use if no other indication + + Returns: + Integer > 0 + """ + return OnnxConfig.default_fixed_num_choices + + @property + def default_onnx_opset(self) -> int: + """ + Which onnx opset to use when exporting the model + + Returns: + Integer ONNX Opset version + """ + return DEFAULT_ONNX_OPSET + + @property + def atol_for_validation(self) -> float: + """ + What absolute tolerance value to use during model conversion validation. + + Returns: + Float absolute tolerance value. + """ + return 1e-5 + + @property + def is_torch_support_available(self) -> bool: + """ + The minimum PyTorch version required to export the model. + + Returns: + `bool`: Whether the installed version of PyTorch is compatible with the model. + """ + if is_torch_available(): + from transformers.utils import get_torch_version + + return version.parse(get_torch_version()) >= self.torch_onnx_minimum_version + else: + return False + + @staticmethod + def use_external_data_format(num_parameters: int) -> bool: + """ + Flag indicating if the model requires using external data format + + Args: + num_parameters: Number of parameter on the model + + Returns: + True if model.num_parameters() * size_of(float32) >= 2Gb False otherwise + """ + + return ( + compute_serialized_parameters_size(num_parameters, ParameterFormat.Float) + >= EXTERNAL_DATA_FORMAT_SIZE_LIMIT + ) + + def _generate_dummy_images( + self, batch_size: int = 2, num_channels: int = 3, image_height: int = 40, image_width: int = 40 + ): + images = [] + for _ in range(batch_size): + data = np.random.rand(image_height, image_width, num_channels) * 255 + images.append(Image.fromarray(data.astype("uint8")).convert("RGB")) + return images + + def _generate_dummy_audio( + self, batch_size: int = 2, sampling_rate: int = 22050, time_duration: float = 5.0, frequency: int = 220 + ): + audio_data = [] + for _ in range(batch_size): + # time variable + t = np.linspace(0, time_duration, int(time_duration * sampling_rate), endpoint=False) + + # generate pure sine wave at `frequency` Hz + audio_data.append(0.5 * np.sin(2 * np.pi * frequency * t)) + + return audio_data + + def generate_dummy_inputs( + self, + preprocessor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin", "ImageProcessingMixin"], + batch_size: int = -1, + seq_length: int = -1, + num_choices: int = -1, + is_pair: bool = False, + framework: Optional[TensorType] = None, + num_channels: int = 3, + image_width: int = 40, + image_height: int = 40, + sampling_rate: int = 22050, + time_duration: float = 5.0, + frequency: int = 220, + tokenizer: Optional["PreTrainedTokenizerBase"] = None, + ) -> Mapping[str, Any]: + """ + Generate inputs to provide to the ONNX exporter for the specific framework + + Args: + preprocessor: ([`PreTrainedTokenizerBase`], [`FeatureExtractionMixin`], or [`ImageProcessingMixin`]): + The preprocessor associated with this model configuration. + batch_size (`int`, *optional*, defaults to -1): + The batch size to export the model for (-1 means dynamic axis). + num_choices (`int`, *optional*, defaults to -1): + The number of candidate answers provided for multiple choice task (-1 means dynamic axis). + seq_length (`int`, *optional*, defaults to -1): + The sequence length to export the model for (-1 means dynamic axis). + is_pair (`bool`, *optional*, defaults to `False`): + Indicate if the input is a pair (sentence 1, sentence 2) + framework (`TensorType`, *optional*, defaults to `None`): + The framework (PyTorch or TensorFlow) that the tokenizer will generate tensors for. + num_channels (`int`, *optional*, defaults to 3): + The number of channels of the generated images. + image_width (`int`, *optional*, defaults to 40): + The width of the generated images. + image_height (`int`, *optional*, defaults to 40): + The height of the generated images. + sampling_rate (`int`, *optional* defaults to 22050) + The sampling rate for audio data generation. + time_duration (`float`, *optional* defaults to 5.0) + Total seconds of sampling for audio data generation. + frequency (`int`, *optional* defaults to 220) + The desired natural frequency of generated audio. + + Returns: + Mapping[str, Tensor] holding the kwargs to provide to the model's forward function + """ + from ..feature_extraction_utils import FeatureExtractionMixin + from ..image_processing_utils import ImageProcessingMixin + from ..tokenization_utils_base import PreTrainedTokenizerBase + + if isinstance(preprocessor, PreTrainedTokenizerBase) and tokenizer is not None: + raise ValueError("You cannot provide both a tokenizer and a preprocessor to generate dummy inputs.") + if tokenizer is not None: + warnings.warn( + "The `tokenizer` argument is deprecated and will be removed in version 5 of Transformers. Use" + " `preprocessor` instead.", + FutureWarning, + ) + logger.warning("Overwriting the `preprocessor` argument with `tokenizer` to generate dummy inputs.") + preprocessor = tokenizer + if isinstance(preprocessor, PreTrainedTokenizerBase): + # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX + batch_size = compute_effective_axis_dimension( + batch_size, fixed_dimension=OnnxConfig.default_fixed_batch, num_token_to_add=0 + ) + # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX + token_to_add = preprocessor.num_special_tokens_to_add(is_pair) + seq_length = compute_effective_axis_dimension( + seq_length, fixed_dimension=OnnxConfig.default_fixed_sequence, num_token_to_add=token_to_add + ) + # Generate dummy inputs according to compute batch and sequence + input_token = ( + preprocessor.unk_token + if (preprocessor.unk_token is not None and len(preprocessor.unk_token) > 0) + else "0" + ) + dummy_input = [" ".join([input_token]) * seq_length] * batch_size + if self.task == "multiple-choice": + # If dynamic axis (-1) we forward with a fixed dimension of 4 candidate answers to avoid optimizations + # made by ONNX + num_choices = compute_effective_axis_dimension( + num_choices, fixed_dimension=OnnxConfig.default_fixed_num_choices, num_token_to_add=0 + ) + dummy_input = dummy_input * num_choices + # The shape of the tokenized inputs values is [batch_size * num_choices, seq_length] + tokenized_input = preprocessor(dummy_input, text_pair=dummy_input) + # Unflatten the tokenized inputs values expanding it to the shape [batch_size, num_choices, seq_length] + for k, v in tokenized_input.items(): + tokenized_input[k] = [v[i : i + num_choices] for i in range(0, len(v), num_choices)] + return dict(tokenized_input.convert_to_tensors(tensor_type=framework)) + return dict(preprocessor(dummy_input, return_tensors=framework)) + elif isinstance(preprocessor, ImageProcessingMixin): + if preprocessor.model_input_names[0] != "pixel_values": + raise ValueError( + f"The `preprocessor` is an image processor ({preprocessor.__class__.__name__}) and expects" + f' `model_input_names[0]` to be "pixel_values", but got {preprocessor.model_input_names[0]}' + ) + # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX + batch_size = compute_effective_axis_dimension(batch_size, fixed_dimension=OnnxConfig.default_fixed_batch) + dummy_input = self._generate_dummy_images(batch_size, num_channels, image_height, image_width) + return dict(preprocessor(images=dummy_input, return_tensors=framework)) + elif isinstance(preprocessor, FeatureExtractionMixin) and preprocessor.model_input_names[0] == "pixel_values": + # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX + batch_size = compute_effective_axis_dimension(batch_size, fixed_dimension=OnnxConfig.default_fixed_batch) + dummy_input = self._generate_dummy_images(batch_size, num_channels, image_height, image_width) + return dict(preprocessor(images=dummy_input, return_tensors=framework)) + elif ( + isinstance(preprocessor, FeatureExtractionMixin) and preprocessor.model_input_names[0] == "input_features" + ): + # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX + batch_size = compute_effective_axis_dimension(batch_size, fixed_dimension=OnnxConfig.default_fixed_batch) + dummy_input = self._generate_dummy_audio(batch_size, sampling_rate, time_duration, frequency) + return dict(preprocessor(dummy_input, return_tensors=framework)) + else: + raise ValueError( + "Unable to generate dummy inputs for the model. Please provide a tokenizer or a preprocessor." + ) + + def generate_dummy_inputs_onnxruntime(self, reference_model_inputs: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Generate inputs for ONNX Runtime using the reference model inputs. Override this to run inference with seq2seq + models which have the encoder and decoder exported as separate ONNX files. + + Args: + reference_model_inputs ([`Mapping[str, Tensor]`): + Reference inputs for the model. + + Returns: + `Mapping[str, Tensor]`: The mapping holding the kwargs to provide to the model's forward function + """ + return reference_model_inputs + + def patch_ops(self): + for spec in self._patching_specs: + custom_op = spec.custom_op if spec.op_wrapper is None else spec.op_wrapper(spec.custom_op) + setattr(spec.o, spec.name, custom_op) + + def restore_ops(self): + for spec in self._patching_specs: + orig_op = spec.orig_op if spec.op_wrapper is None else spec.op_wrapper(spec.orig_op) + setattr(spec.o, spec.name, orig_op) + + @classmethod + def flatten_output_collection_property(cls, name: str, field: Iterable[Any]) -> Dict[str, Any]: + """ + Flatten any potential nested structure expanding the name of the field with the index of the element within the + structure. + + Args: + name: The name of the nested structure + field: The structure to, potentially, be flattened + + Returns: + (Dict[str, Any]): Outputs with flattened structure and key mapping this new structure. + + """ + from itertools import chain + + return {f"{name}.{idx}": item for idx, item in enumerate(chain.from_iterable(field))} + + +class OnnxConfigWithPast(OnnxConfig, ABC): + def __init__( + self, + config: "PretrainedConfig", + task: str = "default", + patching_specs: Optional[list[PatchingSpec]] = None, + use_past: bool = False, + ): + super().__init__(config, task=task, patching_specs=patching_specs) + self.use_past = use_past + + @classmethod + def with_past(cls, config: "PretrainedConfig", task: str = "default") -> "OnnxConfigWithPast": + """ + Instantiate a OnnxConfig with `use_past` attribute set to True + + Args: + config: The underlying model's config to use when exporting to ONNX + + Returns: + OnnxConfig with `.use_past = True` + """ + return cls(config, task=task, use_past=True) + + @property + def outputs(self) -> Mapping[str, Mapping[int, str]]: + common_outputs = super().outputs + if self.use_past: + self.fill_with_past_key_values_(common_outputs, direction="outputs") + + return common_outputs + + @property + def values_override(self) -> Optional[Mapping[str, Any]]: + if hasattr(self._config, "use_cache"): + return {"use_cache": self.use_past} + + return None + + @property + def num_layers(self) -> int: + """ + The number of layers attribute retrieved from the model config. Override this for model configs where the + number of layers attribute is not called `num_layers`. + """ + if not hasattr(self._config, "num_layers"): + raise AttributeError( + "could not find the number of layers attribute in the model configuration, override the num_layers" + " property of the model OnnxConfig to solve this" + ) + return self._config.num_layers + + @property + def num_attention_heads(self) -> int: + """ + The number of attention heads attribute retrieved from the model config. Override this for model configs where + the number of attention heads attribute is not called `num_attention_heads`. + """ + if not hasattr(self._config, "num_attention_heads"): + raise AttributeError( + "could not find the number of attention heads attribute in the model configuration, override the" + " num_attention_heads property of the model OnnxConfig to solve this" + ) + return self._config.num_attention_heads + + def generate_dummy_inputs( + self, + tokenizer: "PreTrainedTokenizerBase", + batch_size: int = -1, + seq_length: int = -1, + is_pair: bool = False, + framework: Optional[TensorType] = None, + ) -> Mapping[str, Any]: + # TODO: should we set seq_length = 1 when self.use_past = True? + common_inputs = super().generate_dummy_inputs( + tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework + ) + + if self.use_past: + if not is_torch_available(): + raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.") + else: + import torch + + batch, seqlen = common_inputs["input_ids"].shape + # Not using the same length for past_key_values + past_key_values_length = seqlen + 2 + shape = ( + batch, + self.num_attention_heads, + past_key_values_length, + self._config.hidden_size // self.num_attention_heads, + ) + + if "attention_mask" in common_inputs: + mask_dtype = common_inputs["attention_mask"].dtype + common_inputs["attention_mask"] = torch.cat( + [common_inputs["attention_mask"], torch.ones(batch, past_key_values_length, dtype=mask_dtype)], + dim=1, + ) + + common_inputs["past_key_values"] = [] + for _ in range(self.num_layers): + common_inputs["past_key_values"].append((torch.zeros(shape), torch.zeros(shape))) + + return common_inputs + + def fill_with_past_key_values_( + self, inputs_or_outputs: Mapping[str, Mapping[int, str]], direction: str, inverted_values_shape: bool = False + ): + """ + Fill the input_or_outputs mapping with past_key_values dynamic axes considering. + + Args: + inputs_or_outputs: The mapping to fill. + direction: either "inputs" or "outputs", it specifies whether input_or_outputs is the input mapping or the + output mapping, this is important for axes naming. + inverted_values_shape: + If `True`, store values on dynamic axis 1, else on axis 2. + + """ + if direction not in ["inputs", "outputs"]: + raise ValueError(f'direction must either be "inputs" or "outputs", but {direction} was given') + + name = "past_key_values" if direction == "inputs" else "present" + for i in range(self.num_layers): + inputs_or_outputs[f"{name}.{i}.key"] = {0: "batch", 2: "past_sequence + sequence"} + if inverted_values_shape: + inputs_or_outputs[f"{name}.{i}.value"] = {0: "batch", 1: "past_sequence + sequence"} + else: + inputs_or_outputs[f"{name}.{i}.value"] = {0: "batch", 2: "past_sequence + sequence"} + + def _flatten_past_key_values_(self, flattened_output, name, idx, t): + flattened_output[f"{name}.{idx}.key"] = t[0] + flattened_output[f"{name}.{idx}.value"] = t[1] + + def flatten_output_collection_property(self, name: str, field: Iterable[Any]) -> Dict[str, Any]: + flattened_output = {} + if name in ["present", "past_key_values"]: + for idx, t in enumerate(field): + self._flatten_past_key_values_(flattened_output, name, idx, t) + else: + flattened_output = super().flatten_output_collection_property(name, field) + + return flattened_output + + +class OnnxSeq2SeqConfigWithPast(OnnxConfigWithPast): + @property + def outputs(self) -> Mapping[str, Mapping[int, str]]: + common_outputs = super(OnnxConfigWithPast, self).outputs + # Renaming the outputs axes properly. + for name, axes_names in common_outputs.items(): + sequence_name = "encoder_sequence" if "encoder" in name else "decoder_sequence" + for axis_idx, name in axes_names.items(): + if "sequence" in name: + axes_names[axis_idx] = sequence_name + # We reset the value as the order in common_outputs (OrderedDict) is lost otherwise + else: + axes_names[axis_idx] = name + if self.use_past: + self.fill_with_past_key_values_(common_outputs, direction="outputs") + + return common_outputs + + @property + def num_layers(self) -> Tuple[int]: + try: + num_layers = super().num_layers + num_layers = (num_layers, num_layers) + except AttributeError: + if hasattr(self._config, "encoder_layers") and hasattr(self._config, "decoder_layers"): + num_layers = (self._config.encoder_layers, self._config.decoder_layers) + else: + raise AttributeError( + "could not find the number of encoder and decoder layers attributes in the model configuration," + " override the num_layers property of the model OnnxConfig to solve this" + ) + + return num_layers + + @property + def num_attention_heads(self) -> Tuple[int]: + try: + num_attention_heads = super().num_attention_heads + num_attention_heads = (num_attention_heads, num_attention_heads) + except AttributeError: + if hasattr(self._config, "encoder_attention_heads") and hasattr(self._config, "decoder_attention_heads"): + num_attention_heads = (self._config.encoder_attention_heads, self._config.decoder_attention_heads) + else: + raise AttributeError( + "could not find the number of attention heads for the encoder and the decoder attributes in the" + " model configuration, override the num_attention_heads property of the model OnnxConfig to solve" + " this" + ) + return num_attention_heads + + def generate_dummy_inputs( + self, + tokenizer: Optional["PreTrainedTokenizerBase"], + batch_size: int = -1, + seq_length: int = -1, + is_pair: bool = False, + framework: Optional[TensorType] = None, + ) -> Mapping[str, Any]: + encoder_inputs = super(OnnxConfigWithPast, self).generate_dummy_inputs( + tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework + ) + + # Generate decoder inputs + decoder_seq_length = seq_length if not self.use_past else 1 + decoder_inputs = super(OnnxConfigWithPast, self).generate_dummy_inputs( + tokenizer, batch_size=batch_size, seq_length=decoder_seq_length, is_pair=is_pair, framework=framework + ) + decoder_inputs = {f"decoder_{name}": tensor for name, tensor in decoder_inputs.items()} + common_inputs = dict(**encoder_inputs, **decoder_inputs) + + if self.use_past: + if not is_torch_available(): + raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.") + else: + import torch + batch = common_inputs["input_ids"].shape[0] + encoder_seq_length = common_inputs["input_ids"].shape[1] + decoder_seq_length = common_inputs["decoder_input_ids"].shape[1] + num_encoder_attention_heads, num_decoder_attention_heads = self.num_attention_heads + encoder_shape = ( + batch, + num_encoder_attention_heads, + encoder_seq_length, + self._config.hidden_size // num_encoder_attention_heads, + ) + decoder_shape = ( + batch, + num_decoder_attention_heads, + # Not using the same length for past_key_values + decoder_seq_length + 3, + self._config.hidden_size // num_decoder_attention_heads, + ) + + common_inputs["past_key_values"] = [] + # If the number of encoder and decoder layers are present in the model configuration, both are considered + num_encoder_layers, num_decoder_layers = self.num_layers + min_num_layers = min(num_encoder_layers, num_decoder_layers) + max_num_layers = max(num_encoder_layers, num_decoder_layers) - min_num_layers + remaining_side_name = "encoder" if num_encoder_layers > num_decoder_layers else "decoder" + + for _ in range(min_num_layers): + # For encoder-decoder models, past_key_values contains pre-computed values for both the encoder and the + # decoder layers, hence a tuple of 4 tensors instead of 2 + common_inputs["past_key_values"].append( + ( + torch.zeros(decoder_shape), + torch.zeros(decoder_shape), + torch.zeros(encoder_shape), + torch.zeros(encoder_shape), + ) + ) + + # TODO: test this. + shape = encoder_shape if remaining_side_name == "encoder" else decoder_shape + for _ in range(min_num_layers, max_num_layers): + common_inputs["past_key_values"].append((torch.zeros(shape), torch.zeros(shape))) + + return common_inputs + + def fill_with_past_key_values_(self, inputs_or_outputs: Mapping[str, Mapping[int, str]], direction: str): + if direction not in ["inputs", "outputs"]: + raise ValueError(f'direction must either be "inputs" or "outputs", but {direction} was given') + + name = "past_key_values" if direction == "inputs" else "present" + + # If the number of encoder and decoder layers are present in the model configuration, both are considered + num_encoder_layers, num_decoder_layers = self.num_layers + min_num_layers = min(num_encoder_layers, num_decoder_layers) + max_num_layers = max(num_encoder_layers, num_decoder_layers) - min_num_layers + remaining_side_name = "encoder" if num_encoder_layers > num_decoder_layers else "decoder" + + encoder_sequence = "past_encoder_sequence" + decoder_sequence = "past_decoder_sequence" if direction == "inputs" else "past_decoder_sequence + sequence" + + for i in range(min_num_layers): + inputs_or_outputs[f"{name}.{i}.decoder.key"] = {0: "batch", 2: decoder_sequence} + inputs_or_outputs[f"{name}.{i}.decoder.value"] = {0: "batch", 2: decoder_sequence} + inputs_or_outputs[f"{name}.{i}.encoder.key"] = {0: "batch", 2: encoder_sequence} + inputs_or_outputs[f"{name}.{i}.encoder.value"] = {0: "batch", 2: encoder_sequence} + + for i in range(min_num_layers, max_num_layers): + if remaining_side_name == "encoder": + axes_info = {0: "batch", 2: encoder_sequence} + else: + axes_info = {0: "batch", 2: decoder_sequence} + inputs_or_outputs[f"{name}.{i}.{remaining_side_name}.key"] = axes_info + + def _flatten_past_key_values_(self, flattened_output, name, idx, t): + flattened_output[f"{name}.{idx}.decoder.key"] = t[0] + flattened_output[f"{name}.{idx}.decoder.value"] = t[1] + flattened_output[f"{name}.{idx}.encoder.key"] = t[2] + flattened_output[f"{name}.{idx}.encoder.value"] = t[3] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_auto.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_auto.py new file mode 100644 index 0000000000000000000000000000000000000000..82ce72cf7b0f776e29ed457180431a1939e514ea --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_auto.py @@ -0,0 +1,1172 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Auto Config class.""" + +import importlib +import os +import re +import warnings +from collections import OrderedDict +from typing import List, Union + +from ...configuration_utils import PretrainedConfig +from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code +from ...utils import CONFIG_NAME, logging + + +logger = logging.get_logger(__name__) + + +CONFIG_MAPPING_NAMES = OrderedDict( + [ + # Add configs here + ("albert", "AlbertConfig"), + ("align", "AlignConfig"), + ("altclip", "AltCLIPConfig"), + ("aria", "AriaConfig"), + ("aria_text", "AriaTextConfig"), + ("audio-spectrogram-transformer", "ASTConfig"), + ("autoformer", "AutoformerConfig"), + ("aya_vision", "AyaVisionConfig"), + ("bamba", "BambaConfig"), + ("bark", "BarkConfig"), + ("bart", "BartConfig"), + ("beit", "BeitConfig"), + ("bert", "BertConfig"), + ("bert-generation", "BertGenerationConfig"), + ("big_bird", "BigBirdConfig"), + ("bigbird_pegasus", "BigBirdPegasusConfig"), + ("biogpt", "BioGptConfig"), + ("bit", "BitConfig"), + ("blenderbot", "BlenderbotConfig"), + ("blenderbot-small", "BlenderbotSmallConfig"), + ("blip", "BlipConfig"), + ("blip-2", "Blip2Config"), + ("bloom", "BloomConfig"), + ("bridgetower", "BridgeTowerConfig"), + ("bros", "BrosConfig"), + ("camembert", "CamembertConfig"), + ("canine", "CanineConfig"), + ("chameleon", "ChameleonConfig"), + ("chinese_clip", "ChineseCLIPConfig"), + ("chinese_clip_vision_model", "ChineseCLIPVisionConfig"), + ("clap", "ClapConfig"), + ("clip", "CLIPConfig"), + ("clip_text_model", "CLIPTextConfig"), + ("clip_vision_model", "CLIPVisionConfig"), + ("clipseg", "CLIPSegConfig"), + ("clvp", "ClvpConfig"), + ("code_llama", "LlamaConfig"), + ("codegen", "CodeGenConfig"), + ("cohere", "CohereConfig"), + ("cohere2", "Cohere2Config"), + ("colpali", "ColPaliConfig"), + ("conditional_detr", "ConditionalDetrConfig"), + ("convbert", "ConvBertConfig"), + ("convnext", "ConvNextConfig"), + ("convnextv2", "ConvNextV2Config"), + ("cpmant", "CpmAntConfig"), + ("ctrl", "CTRLConfig"), + ("cvt", "CvtConfig"), + ("dab-detr", "DabDetrConfig"), + ("dac", "DacConfig"), + ("data2vec-audio", "Data2VecAudioConfig"), + ("data2vec-text", "Data2VecTextConfig"), + ("data2vec-vision", "Data2VecVisionConfig"), + ("dbrx", "DbrxConfig"), + ("deberta", "DebertaConfig"), + ("deberta-v2", "DebertaV2Config"), + ("decision_transformer", "DecisionTransformerConfig"), + ("deepseek_v3", "DeepseekV3Config"), + ("deformable_detr", "DeformableDetrConfig"), + ("deit", "DeiTConfig"), + ("depth_anything", "DepthAnythingConfig"), + ("depth_pro", "DepthProConfig"), + ("deta", "DetaConfig"), + ("detr", "DetrConfig"), + ("diffllama", "DiffLlamaConfig"), + ("dinat", "DinatConfig"), + ("dinov2", "Dinov2Config"), + ("dinov2_with_registers", "Dinov2WithRegistersConfig"), + ("distilbert", "DistilBertConfig"), + ("donut-swin", "DonutSwinConfig"), + ("dpr", "DPRConfig"), + ("dpt", "DPTConfig"), + ("efficientformer", "EfficientFormerConfig"), + ("efficientnet", "EfficientNetConfig"), + ("electra", "ElectraConfig"), + ("emu3", "Emu3Config"), + ("encodec", "EncodecConfig"), + ("encoder-decoder", "EncoderDecoderConfig"), + ("ernie", "ErnieConfig"), + ("ernie_m", "ErnieMConfig"), + ("esm", "EsmConfig"), + ("falcon", "FalconConfig"), + ("falcon_mamba", "FalconMambaConfig"), + ("fastspeech2_conformer", "FastSpeech2ConformerConfig"), + ("flaubert", "FlaubertConfig"), + ("flava", "FlavaConfig"), + ("fnet", "FNetConfig"), + ("focalnet", "FocalNetConfig"), + ("fsmt", "FSMTConfig"), + ("funnel", "FunnelConfig"), + ("fuyu", "FuyuConfig"), + ("gemma", "GemmaConfig"), + ("gemma2", "Gemma2Config"), + ("gemma3", "Gemma3Config"), + ("gemma3_text", "Gemma3TextConfig"), + ("git", "GitConfig"), + ("glm", "GlmConfig"), + ("glm4", "Glm4Config"), + ("glpn", "GLPNConfig"), + ("got_ocr2", "GotOcr2Config"), + ("gpt-sw3", "GPT2Config"), + ("gpt2", "GPT2Config"), + ("gpt_bigcode", "GPTBigCodeConfig"), + ("gpt_neo", "GPTNeoConfig"), + ("gpt_neox", "GPTNeoXConfig"), + ("gpt_neox_japanese", "GPTNeoXJapaneseConfig"), + ("gptj", "GPTJConfig"), + ("gptsan-japanese", "GPTSanJapaneseConfig"), + ("granite", "GraniteConfig"), + ("granitemoe", "GraniteMoeConfig"), + ("granitemoeshared", "GraniteMoeSharedConfig"), + ("granitevision", "LlavaNextConfig"), + ("graphormer", "GraphormerConfig"), + ("grounding-dino", "GroundingDinoConfig"), + ("groupvit", "GroupViTConfig"), + ("helium", "HeliumConfig"), + ("hiera", "HieraConfig"), + ("hubert", "HubertConfig"), + ("ibert", "IBertConfig"), + ("idefics", "IdeficsConfig"), + ("idefics2", "Idefics2Config"), + ("idefics3", "Idefics3Config"), + ("idefics3_vision", "Idefics3VisionConfig"), + ("ijepa", "IJepaConfig"), + ("imagegpt", "ImageGPTConfig"), + ("informer", "InformerConfig"), + ("instructblip", "InstructBlipConfig"), + ("instructblipvideo", "InstructBlipVideoConfig"), + ("jamba", "JambaConfig"), + ("jetmoe", "JetMoeConfig"), + ("jukebox", "JukeboxConfig"), + ("kosmos-2", "Kosmos2Config"), + ("layoutlm", "LayoutLMConfig"), + ("layoutlmv2", "LayoutLMv2Config"), + ("layoutlmv3", "LayoutLMv3Config"), + ("led", "LEDConfig"), + ("levit", "LevitConfig"), + ("lilt", "LiltConfig"), + ("llama", "LlamaConfig"), + ("llama4", "Llama4Config"), + ("llama4_text", "Llama4TextConfig"), + ("llava", "LlavaConfig"), + ("llava_next", "LlavaNextConfig"), + ("llava_next_video", "LlavaNextVideoConfig"), + ("llava_onevision", "LlavaOnevisionConfig"), + ("longformer", "LongformerConfig"), + ("longt5", "LongT5Config"), + ("luke", "LukeConfig"), + ("lxmert", "LxmertConfig"), + ("m2m_100", "M2M100Config"), + ("mamba", "MambaConfig"), + ("mamba2", "Mamba2Config"), + ("marian", "MarianConfig"), + ("markuplm", "MarkupLMConfig"), + ("mask2former", "Mask2FormerConfig"), + ("maskformer", "MaskFormerConfig"), + ("maskformer-swin", "MaskFormerSwinConfig"), + ("mbart", "MBartConfig"), + ("mctct", "MCTCTConfig"), + ("mega", "MegaConfig"), + ("megatron-bert", "MegatronBertConfig"), + ("mgp-str", "MgpstrConfig"), + ("mimi", "MimiConfig"), + ("mistral", "MistralConfig"), + ("mistral3", "Mistral3Config"), + ("mixtral", "MixtralConfig"), + ("mllama", "MllamaConfig"), + ("mobilebert", "MobileBertConfig"), + ("mobilenet_v1", "MobileNetV1Config"), + ("mobilenet_v2", "MobileNetV2Config"), + ("mobilevit", "MobileViTConfig"), + ("mobilevitv2", "MobileViTV2Config"), + ("modernbert", "ModernBertConfig"), + ("moonshine", "MoonshineConfig"), + ("moshi", "MoshiConfig"), + ("mpnet", "MPNetConfig"), + ("mpt", "MptConfig"), + ("mra", "MraConfig"), + ("mt5", "MT5Config"), + ("musicgen", "MusicgenConfig"), + ("musicgen_melody", "MusicgenMelodyConfig"), + ("mvp", "MvpConfig"), + ("nat", "NatConfig"), + ("nemotron", "NemotronConfig"), + ("nezha", "NezhaConfig"), + ("nllb-moe", "NllbMoeConfig"), + ("nougat", "VisionEncoderDecoderConfig"), + ("nystromformer", "NystromformerConfig"), + ("olmo", "OlmoConfig"), + ("olmo2", "Olmo2Config"), + ("olmoe", "OlmoeConfig"), + ("omdet-turbo", "OmDetTurboConfig"), + ("oneformer", "OneFormerConfig"), + ("open-llama", "OpenLlamaConfig"), + ("openai-gpt", "OpenAIGPTConfig"), + ("opt", "OPTConfig"), + ("owlv2", "Owlv2Config"), + ("owlvit", "OwlViTConfig"), + ("paligemma", "PaliGemmaConfig"), + ("patchtsmixer", "PatchTSMixerConfig"), + ("patchtst", "PatchTSTConfig"), + ("pegasus", "PegasusConfig"), + ("pegasus_x", "PegasusXConfig"), + ("perceiver", "PerceiverConfig"), + ("persimmon", "PersimmonConfig"), + ("phi", "PhiConfig"), + ("phi3", "Phi3Config"), + ("phi4_multimodal", "Phi4MultimodalConfig"), + ("phimoe", "PhimoeConfig"), + ("pix2struct", "Pix2StructConfig"), + ("pixtral", "PixtralVisionConfig"), + ("plbart", "PLBartConfig"), + ("poolformer", "PoolFormerConfig"), + ("pop2piano", "Pop2PianoConfig"), + ("prompt_depth_anything", "PromptDepthAnythingConfig"), + ("prophetnet", "ProphetNetConfig"), + ("pvt", "PvtConfig"), + ("pvt_v2", "PvtV2Config"), + ("qdqbert", "QDQBertConfig"), + ("qwen2", "Qwen2Config"), + ("qwen2_5_vl", "Qwen2_5_VLConfig"), + ("qwen2_audio", "Qwen2AudioConfig"), + ("qwen2_audio_encoder", "Qwen2AudioEncoderConfig"), + ("qwen2_moe", "Qwen2MoeConfig"), + ("qwen2_vl", "Qwen2VLConfig"), + ("qwen3", "Qwen3Config"), + ("qwen3_moe", "Qwen3MoeConfig"), + ("rag", "RagConfig"), + ("realm", "RealmConfig"), + ("recurrent_gemma", "RecurrentGemmaConfig"), + ("reformer", "ReformerConfig"), + ("regnet", "RegNetConfig"), + ("rembert", "RemBertConfig"), + ("resnet", "ResNetConfig"), + ("retribert", "RetriBertConfig"), + ("roberta", "RobertaConfig"), + ("roberta-prelayernorm", "RobertaPreLayerNormConfig"), + ("roc_bert", "RoCBertConfig"), + ("roformer", "RoFormerConfig"), + ("rt_detr", "RTDetrConfig"), + ("rt_detr_resnet", "RTDetrResNetConfig"), + ("rt_detr_v2", "RTDetrV2Config"), + ("rwkv", "RwkvConfig"), + ("sam", "SamConfig"), + ("sam_vision_model", "SamVisionConfig"), + ("seamless_m4t", "SeamlessM4TConfig"), + ("seamless_m4t_v2", "SeamlessM4Tv2Config"), + ("segformer", "SegformerConfig"), + ("seggpt", "SegGptConfig"), + ("sew", "SEWConfig"), + ("sew-d", "SEWDConfig"), + ("shieldgemma2", "ShieldGemma2Config"), + ("siglip", "SiglipConfig"), + ("siglip2", "Siglip2Config"), + ("siglip_vision_model", "SiglipVisionConfig"), + ("smolvlm", "SmolVLMConfig"), + ("smolvlm_vision", "SmolVLMVisionConfig"), + ("speech-encoder-decoder", "SpeechEncoderDecoderConfig"), + ("speech_to_text", "Speech2TextConfig"), + ("speech_to_text_2", "Speech2Text2Config"), + ("speecht5", "SpeechT5Config"), + ("splinter", "SplinterConfig"), + ("squeezebert", "SqueezeBertConfig"), + ("stablelm", "StableLmConfig"), + ("starcoder2", "Starcoder2Config"), + ("superglue", "SuperGlueConfig"), + ("superpoint", "SuperPointConfig"), + ("swiftformer", "SwiftFormerConfig"), + ("swin", "SwinConfig"), + ("swin2sr", "Swin2SRConfig"), + ("swinv2", "Swinv2Config"), + ("switch_transformers", "SwitchTransformersConfig"), + ("t5", "T5Config"), + ("table-transformer", "TableTransformerConfig"), + ("tapas", "TapasConfig"), + ("textnet", "TextNetConfig"), + ("time_series_transformer", "TimeSeriesTransformerConfig"), + ("timesformer", "TimesformerConfig"), + ("timm_backbone", "TimmBackboneConfig"), + ("timm_wrapper", "TimmWrapperConfig"), + ("trajectory_transformer", "TrajectoryTransformerConfig"), + ("transfo-xl", "TransfoXLConfig"), + ("trocr", "TrOCRConfig"), + ("tvlt", "TvltConfig"), + ("tvp", "TvpConfig"), + ("udop", "UdopConfig"), + ("umt5", "UMT5Config"), + ("unispeech", "UniSpeechConfig"), + ("unispeech-sat", "UniSpeechSatConfig"), + ("univnet", "UnivNetConfig"), + ("upernet", "UperNetConfig"), + ("van", "VanConfig"), + ("video_llava", "VideoLlavaConfig"), + ("videomae", "VideoMAEConfig"), + ("vilt", "ViltConfig"), + ("vipllava", "VipLlavaConfig"), + ("vision-encoder-decoder", "VisionEncoderDecoderConfig"), + ("vision-text-dual-encoder", "VisionTextDualEncoderConfig"), + ("visual_bert", "VisualBertConfig"), + ("vit", "ViTConfig"), + ("vit_hybrid", "ViTHybridConfig"), + ("vit_mae", "ViTMAEConfig"), + ("vit_msn", "ViTMSNConfig"), + ("vitdet", "VitDetConfig"), + ("vitmatte", "VitMatteConfig"), + ("vitpose", "VitPoseConfig"), + ("vitpose_backbone", "VitPoseBackboneConfig"), + ("vits", "VitsConfig"), + ("vivit", "VivitConfig"), + ("wav2vec2", "Wav2Vec2Config"), + ("wav2vec2-bert", "Wav2Vec2BertConfig"), + ("wav2vec2-conformer", "Wav2Vec2ConformerConfig"), + ("wavlm", "WavLMConfig"), + ("whisper", "WhisperConfig"), + ("xclip", "XCLIPConfig"), + ("xglm", "XGLMConfig"), + ("xlm", "XLMConfig"), + ("xlm-prophetnet", "XLMProphetNetConfig"), + ("xlm-roberta", "XLMRobertaConfig"), + ("xlm-roberta-xl", "XLMRobertaXLConfig"), + ("xlnet", "XLNetConfig"), + ("xmod", "XmodConfig"), + ("yolos", "YolosConfig"), + ("yoso", "YosoConfig"), + ("zamba", "ZambaConfig"), + ("zamba2", "Zamba2Config"), + ("zoedepth", "ZoeDepthConfig"), + ] +) + + +MODEL_NAMES_MAPPING = OrderedDict( + [ + # Add full (and cased) model names here + ("albert", "ALBERT"), + ("align", "ALIGN"), + ("altclip", "AltCLIP"), + ("aria", "Aria"), + ("aria_text", "AriaText"), + ("audio-spectrogram-transformer", "Audio Spectrogram Transformer"), + ("autoformer", "Autoformer"), + ("aya_vision", "AyaVision"), + ("bamba", "Bamba"), + ("bark", "Bark"), + ("bart", "BART"), + ("barthez", "BARThez"), + ("bartpho", "BARTpho"), + ("beit", "BEiT"), + ("bert", "BERT"), + ("bert-generation", "Bert Generation"), + ("bert-japanese", "BertJapanese"), + ("bertweet", "BERTweet"), + ("big_bird", "BigBird"), + ("bigbird_pegasus", "BigBird-Pegasus"), + ("biogpt", "BioGpt"), + ("bit", "BiT"), + ("blenderbot", "Blenderbot"), + ("blenderbot-small", "BlenderbotSmall"), + ("blip", "BLIP"), + ("blip-2", "BLIP-2"), + ("bloom", "BLOOM"), + ("bort", "BORT"), + ("bridgetower", "BridgeTower"), + ("bros", "BROS"), + ("byt5", "ByT5"), + ("camembert", "CamemBERT"), + ("canine", "CANINE"), + ("chameleon", "Chameleon"), + ("chinese_clip", "Chinese-CLIP"), + ("chinese_clip_vision_model", "ChineseCLIPVisionModel"), + ("clap", "CLAP"), + ("clip", "CLIP"), + ("clip_text_model", "CLIPTextModel"), + ("clip_vision_model", "CLIPVisionModel"), + ("clipseg", "CLIPSeg"), + ("clvp", "CLVP"), + ("code_llama", "CodeLlama"), + ("codegen", "CodeGen"), + ("cohere", "Cohere"), + ("cohere2", "Cohere2"), + ("colpali", "ColPali"), + ("conditional_detr", "Conditional DETR"), + ("convbert", "ConvBERT"), + ("convnext", "ConvNeXT"), + ("convnextv2", "ConvNeXTV2"), + ("cpm", "CPM"), + ("cpmant", "CPM-Ant"), + ("ctrl", "CTRL"), + ("cvt", "CvT"), + ("dab-detr", "DAB-DETR"), + ("dac", "DAC"), + ("data2vec-audio", "Data2VecAudio"), + ("data2vec-text", "Data2VecText"), + ("data2vec-vision", "Data2VecVision"), + ("dbrx", "DBRX"), + ("deberta", "DeBERTa"), + ("deberta-v2", "DeBERTa-v2"), + ("decision_transformer", "Decision Transformer"), + ("deepseek_v3", "DeepSeek-V3"), + ("deformable_detr", "Deformable DETR"), + ("deit", "DeiT"), + ("deplot", "DePlot"), + ("depth_anything", "Depth Anything"), + ("depth_anything_v2", "Depth Anything V2"), + ("depth_pro", "DepthPro"), + ("deta", "DETA"), + ("detr", "DETR"), + ("dialogpt", "DialoGPT"), + ("diffllama", "DiffLlama"), + ("dinat", "DiNAT"), + ("dinov2", "DINOv2"), + ("dinov2_with_registers", "DINOv2 with Registers"), + ("distilbert", "DistilBERT"), + ("dit", "DiT"), + ("donut-swin", "DonutSwin"), + ("dpr", "DPR"), + ("dpt", "DPT"), + ("efficientformer", "EfficientFormer"), + ("efficientnet", "EfficientNet"), + ("electra", "ELECTRA"), + ("emu3", "Emu3"), + ("encodec", "EnCodec"), + ("encoder-decoder", "Encoder decoder"), + ("ernie", "ERNIE"), + ("ernie_m", "ErnieM"), + ("esm", "ESM"), + ("falcon", "Falcon"), + ("falcon3", "Falcon3"), + ("falcon_mamba", "FalconMamba"), + ("fastspeech2_conformer", "FastSpeech2Conformer"), + ("flan-t5", "FLAN-T5"), + ("flan-ul2", "FLAN-UL2"), + ("flaubert", "FlauBERT"), + ("flava", "FLAVA"), + ("fnet", "FNet"), + ("focalnet", "FocalNet"), + ("fsmt", "FairSeq Machine-Translation"), + ("funnel", "Funnel Transformer"), + ("fuyu", "Fuyu"), + ("gemma", "Gemma"), + ("gemma2", "Gemma2"), + ("gemma3", "Gemma3ForConditionalGeneration"), + ("gemma3_text", "Gemma3ForCausalLM"), + ("git", "GIT"), + ("glm", "GLM"), + ("glm4", "glm4"), + ("glpn", "GLPN"), + ("got_ocr2", "GOT-OCR2"), + ("gpt-sw3", "GPT-Sw3"), + ("gpt2", "OpenAI GPT-2"), + ("gpt_bigcode", "GPTBigCode"), + ("gpt_neo", "GPT Neo"), + ("gpt_neox", "GPT NeoX"), + ("gpt_neox_japanese", "GPT NeoX Japanese"), + ("gptj", "GPT-J"), + ("gptsan-japanese", "GPTSAN-japanese"), + ("granite", "Granite"), + ("granitemoe", "GraniteMoeMoe"), + ("granitemoeshared", "GraniteMoeSharedMoe"), + ("granitevision", "LLaVA-NeXT"), + ("graphormer", "Graphormer"), + ("grounding-dino", "Grounding DINO"), + ("groupvit", "GroupViT"), + ("helium", "Helium"), + ("herbert", "HerBERT"), + ("hiera", "Hiera"), + ("hubert", "Hubert"), + ("ibert", "I-BERT"), + ("idefics", "IDEFICS"), + ("idefics2", "Idefics2"), + ("idefics3", "Idefics3"), + ("idefics3_vision", "Idefics3VisionTransformer"), + ("ijepa", "I-JEPA"), + ("imagegpt", "ImageGPT"), + ("informer", "Informer"), + ("instructblip", "InstructBLIP"), + ("instructblipvideo", "InstructBlipVideo"), + ("jamba", "Jamba"), + ("jetmoe", "JetMoe"), + ("jukebox", "Jukebox"), + ("kosmos-2", "KOSMOS-2"), + ("layoutlm", "LayoutLM"), + ("layoutlmv2", "LayoutLMv2"), + ("layoutlmv3", "LayoutLMv3"), + ("layoutxlm", "LayoutXLM"), + ("led", "LED"), + ("levit", "LeViT"), + ("lilt", "LiLT"), + ("llama", "LLaMA"), + ("llama2", "Llama2"), + ("llama3", "Llama3"), + ("llama4", "Llama4"), + ("llama4_text", "Llama4ForCausalLM"), + ("llava", "LLaVa"), + ("llava_next", "LLaVA-NeXT"), + ("llava_next_video", "LLaVa-NeXT-Video"), + ("llava_onevision", "LLaVA-Onevision"), + ("longformer", "Longformer"), + ("longt5", "LongT5"), + ("luke", "LUKE"), + ("lxmert", "LXMERT"), + ("m2m_100", "M2M100"), + ("madlad-400", "MADLAD-400"), + ("mamba", "Mamba"), + ("mamba2", "mamba2"), + ("marian", "Marian"), + ("markuplm", "MarkupLM"), + ("mask2former", "Mask2Former"), + ("maskformer", "MaskFormer"), + ("maskformer-swin", "MaskFormerSwin"), + ("matcha", "MatCha"), + ("mbart", "mBART"), + ("mbart50", "mBART-50"), + ("mctct", "M-CTC-T"), + ("mega", "MEGA"), + ("megatron-bert", "Megatron-BERT"), + ("megatron_gpt2", "Megatron-GPT2"), + ("mgp-str", "MGP-STR"), + ("mimi", "Mimi"), + ("mistral", "Mistral"), + ("mistral3", "Mistral3"), + ("mixtral", "Mixtral"), + ("mllama", "Mllama"), + ("mluke", "mLUKE"), + ("mms", "MMS"), + ("mobilebert", "MobileBERT"), + ("mobilenet_v1", "MobileNetV1"), + ("mobilenet_v2", "MobileNetV2"), + ("mobilevit", "MobileViT"), + ("mobilevitv2", "MobileViTV2"), + ("modernbert", "ModernBERT"), + ("moonshine", "Moonshine"), + ("moshi", "Moshi"), + ("mpnet", "MPNet"), + ("mpt", "MPT"), + ("mra", "MRA"), + ("mt5", "MT5"), + ("musicgen", "MusicGen"), + ("musicgen_melody", "MusicGen Melody"), + ("mvp", "MVP"), + ("myt5", "myt5"), + ("nat", "NAT"), + ("nemotron", "Nemotron"), + ("nezha", "Nezha"), + ("nllb", "NLLB"), + ("nllb-moe", "NLLB-MOE"), + ("nougat", "Nougat"), + ("nystromformer", "Nyströmformer"), + ("olmo", "OLMo"), + ("olmo2", "OLMo2"), + ("olmoe", "OLMoE"), + ("omdet-turbo", "OmDet-Turbo"), + ("oneformer", "OneFormer"), + ("open-llama", "OpenLlama"), + ("openai-gpt", "OpenAI GPT"), + ("opt", "OPT"), + ("owlv2", "OWLv2"), + ("owlvit", "OWL-ViT"), + ("paligemma", "PaliGemma"), + ("patchtsmixer", "PatchTSMixer"), + ("patchtst", "PatchTST"), + ("pegasus", "Pegasus"), + ("pegasus_x", "PEGASUS-X"), + ("perceiver", "Perceiver"), + ("persimmon", "Persimmon"), + ("phi", "Phi"), + ("phi3", "Phi3"), + ("phi4_multimodal", "Phi4Multimodal"), + ("phimoe", "Phimoe"), + ("phobert", "PhoBERT"), + ("pix2struct", "Pix2Struct"), + ("pixtral", "Pixtral"), + ("plbart", "PLBart"), + ("poolformer", "PoolFormer"), + ("pop2piano", "Pop2Piano"), + ("prompt_depth_anything", "PromptDepthAnything"), + ("prophetnet", "ProphetNet"), + ("pvt", "PVT"), + ("pvt_v2", "PVTv2"), + ("qdqbert", "QDQBert"), + ("qwen2", "Qwen2"), + ("qwen2_5_vl", "Qwen2_5_VL"), + ("qwen2_audio", "Qwen2Audio"), + ("qwen2_audio_encoder", "Qwen2AudioEncoder"), + ("qwen2_moe", "Qwen2MoE"), + ("qwen2_vl", "Qwen2VL"), + ("qwen3", "Qwen3"), + ("qwen3_moe", "Qwen3MoE"), + ("rag", "RAG"), + ("realm", "REALM"), + ("recurrent_gemma", "RecurrentGemma"), + ("reformer", "Reformer"), + ("regnet", "RegNet"), + ("rembert", "RemBERT"), + ("resnet", "ResNet"), + ("retribert", "RetriBERT"), + ("roberta", "RoBERTa"), + ("roberta-prelayernorm", "RoBERTa-PreLayerNorm"), + ("roc_bert", "RoCBert"), + ("roformer", "RoFormer"), + ("rt_detr", "RT-DETR"), + ("rt_detr_resnet", "RT-DETR-ResNet"), + ("rt_detr_v2", "RT-DETRv2"), + ("rwkv", "RWKV"), + ("sam", "SAM"), + ("sam_vision_model", "SamVisionModel"), + ("seamless_m4t", "SeamlessM4T"), + ("seamless_m4t_v2", "SeamlessM4Tv2"), + ("segformer", "SegFormer"), + ("seggpt", "SegGPT"), + ("sew", "SEW"), + ("sew-d", "SEW-D"), + ("shieldgemma2", "Shieldgemma2"), + ("siglip", "SigLIP"), + ("siglip2", "SigLIP2"), + ("siglip2_vision_model", "Siglip2VisionModel"), + ("siglip_vision_model", "SiglipVisionModel"), + ("smolvlm", "SmolVLM"), + ("smolvlm_vision", "SmolVLMVisionTransformer"), + ("speech-encoder-decoder", "Speech Encoder decoder"), + ("speech_to_text", "Speech2Text"), + ("speech_to_text_2", "Speech2Text2"), + ("speecht5", "SpeechT5"), + ("splinter", "Splinter"), + ("squeezebert", "SqueezeBERT"), + ("stablelm", "StableLm"), + ("starcoder2", "Starcoder2"), + ("superglue", "SuperGlue"), + ("superpoint", "SuperPoint"), + ("swiftformer", "SwiftFormer"), + ("swin", "Swin Transformer"), + ("swin2sr", "Swin2SR"), + ("swinv2", "Swin Transformer V2"), + ("switch_transformers", "SwitchTransformers"), + ("t5", "T5"), + ("t5v1.1", "T5v1.1"), + ("table-transformer", "Table Transformer"), + ("tapas", "TAPAS"), + ("tapex", "TAPEX"), + ("textnet", "TextNet"), + ("time_series_transformer", "Time Series Transformer"), + ("timesformer", "TimeSformer"), + ("timm_backbone", "TimmBackbone"), + ("timm_wrapper", "TimmWrapperModel"), + ("trajectory_transformer", "Trajectory Transformer"), + ("transfo-xl", "Transformer-XL"), + ("trocr", "TrOCR"), + ("tvlt", "TVLT"), + ("tvp", "TVP"), + ("udop", "UDOP"), + ("ul2", "UL2"), + ("umt5", "UMT5"), + ("unispeech", "UniSpeech"), + ("unispeech-sat", "UniSpeechSat"), + ("univnet", "UnivNet"), + ("upernet", "UPerNet"), + ("van", "VAN"), + ("video_llava", "VideoLlava"), + ("videomae", "VideoMAE"), + ("vilt", "ViLT"), + ("vipllava", "VipLlava"), + ("vision-encoder-decoder", "Vision Encoder decoder"), + ("vision-text-dual-encoder", "VisionTextDualEncoder"), + ("visual_bert", "VisualBERT"), + ("vit", "ViT"), + ("vit_hybrid", "ViT Hybrid"), + ("vit_mae", "ViTMAE"), + ("vit_msn", "ViTMSN"), + ("vitdet", "VitDet"), + ("vitmatte", "ViTMatte"), + ("vitpose", "ViTPose"), + ("vitpose_backbone", "ViTPoseBackbone"), + ("vits", "VITS"), + ("vivit", "ViViT"), + ("wav2vec2", "Wav2Vec2"), + ("wav2vec2-bert", "Wav2Vec2-BERT"), + ("wav2vec2-conformer", "Wav2Vec2-Conformer"), + ("wav2vec2_phoneme", "Wav2Vec2Phoneme"), + ("wavlm", "WavLM"), + ("whisper", "Whisper"), + ("xclip", "X-CLIP"), + ("xglm", "XGLM"), + ("xlm", "XLM"), + ("xlm-prophetnet", "XLM-ProphetNet"), + ("xlm-roberta", "XLM-RoBERTa"), + ("xlm-roberta-xl", "XLM-RoBERTa-XL"), + ("xlm-v", "XLM-V"), + ("xlnet", "XLNet"), + ("xls_r", "XLS-R"), + ("xlsr_wav2vec2", "XLSR-Wav2Vec2"), + ("xmod", "X-MOD"), + ("yolos", "YOLOS"), + ("yoso", "YOSO"), + ("zamba", "Zamba"), + ("zamba2", "Zamba2"), + ("zoedepth", "ZoeDepth"), + ] +) + +# This is tied to the processing `-` -> `_` in `model_type_to_module_name`. For example, instead of putting +# `transfo-xl` (as in `CONFIG_MAPPING_NAMES`), we should use `transfo_xl`. +DEPRECATED_MODELS = [ + "bort", + "deta", + "efficientformer", + "ernie_m", + "gptsan_japanese", + "graphormer", + "jukebox", + "mctct", + "mega", + "mmbt", + "nat", + "nezha", + "open_llama", + "qdqbert", + "realm", + "retribert", + "speech_to_text_2", + "tapex", + "trajectory_transformer", + "transfo_xl", + "tvlt", + "van", + "vit_hybrid", + "xlm_prophetnet", +] + +SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict( + [ + ("openai-gpt", "openai"), + ("data2vec-audio", "data2vec"), + ("data2vec-text", "data2vec"), + ("data2vec-vision", "data2vec"), + ("donut-swin", "donut"), + ("kosmos-2", "kosmos2"), + ("maskformer-swin", "maskformer"), + ("xclip", "x_clip"), + ("clip_vision_model", "clip"), + ("qwen2_audio_encoder", "qwen2_audio"), + ("clip_text_model", "clip"), + ("aria_text", "aria"), + ("gemma3_text", "gemma3"), + ("idefics3_vision", "idefics3"), + ("siglip_vision_model", "siglip"), + ("smolvlm_vision", "smolvlm"), + ("chinese_clip_vision_model", "chinese_clip"), + ("rt_detr_resnet", "rt_detr"), + ("granitevision", "llava_next"), + ("sam_vision_model", "sam"), + ("llama4_text", "llama4"), + ] +) + + +def model_type_to_module_name(key): + """Converts a config key to the corresponding module.""" + # Special treatment + if key in SPECIAL_MODEL_TYPE_TO_MODULE_NAME: + key = SPECIAL_MODEL_TYPE_TO_MODULE_NAME[key] + + if key in DEPRECATED_MODELS: + key = f"deprecated.{key}" + return key + + key = key.replace("-", "_") + if key in DEPRECATED_MODELS: + key = f"deprecated.{key}" + + return key + + +def config_class_to_model_type(config): + """Converts a config class name to the corresponding model type""" + for key, cls in CONFIG_MAPPING_NAMES.items(): + if cls == config: + return key + # if key not found check in extra content + for key, cls in CONFIG_MAPPING._extra_content.items(): + if cls.__name__ == config: + return key + return None + + +class _LazyConfigMapping(OrderedDict): + """ + A dictionary that lazily load its values when they are requested. + """ + + def __init__(self, mapping): + self._mapping = mapping + self._extra_content = {} + self._modules = {} + + def __getitem__(self, key): + if key in self._extra_content: + return self._extra_content[key] + if key not in self._mapping: + raise KeyError(key) + value = self._mapping[key] + module_name = model_type_to_module_name(key) + if module_name not in self._modules: + self._modules[module_name] = importlib.import_module(f".{module_name}", "transformers.models") + if hasattr(self._modules[module_name], value): + return getattr(self._modules[module_name], value) + + # Some of the mappings have entries model_type -> config of another model type. In that case we try to grab the + # object at the top level. + transformers_module = importlib.import_module("transformers") + return getattr(transformers_module, value) + + def keys(self): + return list(self._mapping.keys()) + list(self._extra_content.keys()) + + def values(self): + return [self[k] for k in self._mapping.keys()] + list(self._extra_content.values()) + + def items(self): + return [(k, self[k]) for k in self._mapping.keys()] + list(self._extra_content.items()) + + def __iter__(self): + return iter(list(self._mapping.keys()) + list(self._extra_content.keys())) + + def __contains__(self, item): + return item in self._mapping or item in self._extra_content + + def register(self, key, value, exist_ok=False): + """ + Register a new configuration in this mapping. + """ + if key in self._mapping.keys() and not exist_ok: + raise ValueError(f"'{key}' is already used by a Transformers config, pick another name.") + self._extra_content[key] = value + + +CONFIG_MAPPING = _LazyConfigMapping(CONFIG_MAPPING_NAMES) + + +class _LazyLoadAllMappings(OrderedDict): + """ + A mapping that will load all pairs of key values at the first access (either by indexing, requestions keys, values, + etc.) + + Args: + mapping: The mapping to load. + """ + + def __init__(self, mapping): + self._mapping = mapping + self._initialized = False + self._data = {} + + def _initialize(self): + if self._initialized: + return + + for model_type, map_name in self._mapping.items(): + module_name = model_type_to_module_name(model_type) + module = importlib.import_module(f".{module_name}", "transformers.models") + mapping = getattr(module, map_name) + self._data.update(mapping) + + self._initialized = True + + def __getitem__(self, key): + self._initialize() + return self._data[key] + + def keys(self): + self._initialize() + return self._data.keys() + + def values(self): + self._initialize() + return self._data.values() + + def items(self): + self._initialize() + return self._data.keys() + + def __iter__(self): + self._initialize() + return iter(self._data) + + def __contains__(self, item): + self._initialize() + return item in self._data + + +def _get_class_name(model_class: Union[str, List[str]]): + if isinstance(model_class, (list, tuple)): + return " or ".join([f"[`{c}`]" for c in model_class if c is not None]) + return f"[`{model_class}`]" + + +def _list_model_options(indent, config_to_class=None, use_model_types=True): + if config_to_class is None and not use_model_types: + raise ValueError("Using `use_model_types=False` requires a `config_to_class` dictionary.") + if use_model_types: + if config_to_class is None: + model_type_to_name = {model_type: f"[`{config}`]" for model_type, config in CONFIG_MAPPING_NAMES.items()} + else: + model_type_to_name = { + model_type: _get_class_name(model_class) + for model_type, model_class in config_to_class.items() + if model_type in MODEL_NAMES_MAPPING + } + lines = [ + f"{indent}- **{model_type}** -- {model_type_to_name[model_type]} ({MODEL_NAMES_MAPPING[model_type]} model)" + for model_type in sorted(model_type_to_name.keys()) + ] + else: + config_to_name = { + CONFIG_MAPPING_NAMES[config]: _get_class_name(clas) + for config, clas in config_to_class.items() + if config in CONFIG_MAPPING_NAMES + } + config_to_model_name = { + config: MODEL_NAMES_MAPPING[model_type] for model_type, config in CONFIG_MAPPING_NAMES.items() + } + lines = [ + f"{indent}- [`{config_name}`] configuration class:" + f" {config_to_name[config_name]} ({config_to_model_name[config_name]} model)" + for config_name in sorted(config_to_name.keys()) + ] + return "\n".join(lines) + + +def replace_list_option_in_docstrings(config_to_class=None, use_model_types=True): + def docstring_decorator(fn): + docstrings = fn.__doc__ + if docstrings is None: + # Example: -OO + return fn + lines = docstrings.split("\n") + i = 0 + while i < len(lines) and re.search(r"^(\s*)List options\s*$", lines[i]) is None: + i += 1 + if i < len(lines): + indent = re.search(r"^(\s*)List options\s*$", lines[i]).groups()[0] + if use_model_types: + indent = f"{indent} " + lines[i] = _list_model_options(indent, config_to_class=config_to_class, use_model_types=use_model_types) + docstrings = "\n".join(lines) + else: + raise ValueError( + f"The function {fn} should have an empty 'List options' in its docstring as placeholder, current" + f" docstring is:\n{docstrings}" + ) + fn.__doc__ = docstrings + return fn + + return docstring_decorator + + +class AutoConfig: + r""" + This is a generic configuration class that will be instantiated as one of the configuration classes of the library + when created with the [`~AutoConfig.from_pretrained`] class method. + + This class cannot be instantiated directly using `__init__()` (throws an error). + """ + + def __init__(self): + raise EnvironmentError( + "AutoConfig is designed to be instantiated " + "using the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` method." + ) + + @classmethod + def for_model(cls, model_type: str, *args, **kwargs): + if model_type in CONFIG_MAPPING: + config_class = CONFIG_MAPPING[model_type] + return config_class(*args, **kwargs) + raise ValueError( + f"Unrecognized model identifier: {model_type}. Should contain one of {', '.join(CONFIG_MAPPING.keys())}" + ) + + @classmethod + @replace_list_option_in_docstrings() + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" + Instantiate one of the configuration classes of the library from a pretrained model configuration. + + The configuration class to instantiate is selected based on the `model_type` property of the config object that + is loaded, or when it's missing, by falling back to using pattern matching on `pretrained_model_name_or_path`: + + List options + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + Can be either: + + - A string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. + - A path to a *directory* containing a configuration file saved using the + [`~PretrainedConfig.save_pretrained`] method, or the [`~PreTrainedModel.save_pretrained`] method, + e.g., `./my_model_directory/`. + - A path or url to a saved configuration JSON *file*, e.g., + `./my_model_directory/configuration.json`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download the model weights and configuration files and override the + cached versions if they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + If `False`, then this function returns just the final configuration object. + + If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs* is a + dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e., the + part of `kwargs` which has not been used to update `config` and is otherwise ignored. + trust_remote_code (`bool`, *optional*, defaults to `False`): + Whether or not to allow for custom models defined on the Hub in their own modeling files. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + kwargs(additional keyword arguments, *optional*): + The values in kwargs of any keys which are configuration attributes will be used to override the loaded + values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled + by the `return_unused_kwargs` keyword parameter. + + Examples: + + ```python + >>> from transformers import AutoConfig + + >>> # Download configuration from huggingface.co and cache. + >>> config = AutoConfig.from_pretrained("google-bert/bert-base-uncased") + + >>> # Download configuration from huggingface.co (user-uploaded) and cache. + >>> config = AutoConfig.from_pretrained("dbmdz/bert-base-german-cased") + + >>> # If configuration file is in a directory (e.g., was saved using *save_pretrained('./test/saved_model/')*). + >>> config = AutoConfig.from_pretrained("./test/bert_saved_model/") + + >>> # Load a specific configuration file. + >>> config = AutoConfig.from_pretrained("./test/bert_saved_model/my_configuration.json") + + >>> # Change some config attributes when loading a pretrained config. + >>> config = AutoConfig.from_pretrained("google-bert/bert-base-uncased", output_attentions=True, foo=False) + >>> config.output_attentions + True + + >>> config, unused_kwargs = AutoConfig.from_pretrained( + ... "google-bert/bert-base-uncased", output_attentions=True, foo=False, return_unused_kwargs=True + ... ) + >>> config.output_attentions + True + + >>> unused_kwargs + {'foo': False} + ```""" + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if kwargs.get("token", None) is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + kwargs["token"] = use_auth_token + + kwargs["_from_auto"] = True + kwargs["name_or_path"] = pretrained_model_name_or_path + trust_remote_code = kwargs.pop("trust_remote_code", None) + code_revision = kwargs.pop("code_revision", None) + + config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs) + has_remote_code = "auto_map" in config_dict and "AutoConfig" in config_dict["auto_map"] + has_local_code = "model_type" in config_dict and config_dict["model_type"] in CONFIG_MAPPING + trust_remote_code = resolve_trust_remote_code( + trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code + ) + + if has_remote_code and trust_remote_code: + class_ref = config_dict["auto_map"]["AutoConfig"] + config_class = get_class_from_dynamic_module( + class_ref, pretrained_model_name_or_path, code_revision=code_revision, **kwargs + ) + if os.path.isdir(pretrained_model_name_or_path): + config_class.register_for_auto_class() + return config_class.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif "model_type" in config_dict: + try: + config_class = CONFIG_MAPPING[config_dict["model_type"]] + except KeyError: + raise ValueError( + f"The checkpoint you are trying to load has model type `{config_dict['model_type']}` " + "but Transformers does not recognize this architecture. This could be because of an " + "issue with the checkpoint, or because your version of Transformers is out of date.\n\n" + "You can update Transformers with the command `pip install --upgrade transformers`. If this " + "does not work, and the checkpoint is very new, then there may not be a release version " + "that supports this model yet. In this case, you can get the most up-to-date code by installing " + "Transformers from source with the command " + "`pip install git+https://github.com/huggingface/transformers.git`" + ) + return config_class.from_dict(config_dict, **unused_kwargs) + else: + # Fallback: use pattern matching on the string. + # We go from longer names to shorter names to catch roberta before bert (for instance) + for pattern in sorted(CONFIG_MAPPING.keys(), key=len, reverse=True): + if pattern in str(pretrained_model_name_or_path): + return CONFIG_MAPPING[pattern].from_dict(config_dict, **unused_kwargs) + + raise ValueError( + f"Unrecognized model in {pretrained_model_name_or_path}. " + f"Should have a `model_type` key in its {CONFIG_NAME}, or contain one of the following strings " + f"in its name: {', '.join(CONFIG_MAPPING.keys())}" + ) + + @staticmethod + def register(model_type, config, exist_ok=False): + """ + Register a new configuration for this class. + + Args: + model_type (`str`): The model type like "bert" or "gpt". + config ([`PretrainedConfig`]): The config to register. + """ + if issubclass(config, PretrainedConfig) and config.model_type != model_type: + raise ValueError( + "The config you are passing has a `model_type` attribute that is not consistent with the model type " + f"you passed (config has {config.model_type} and you passed {model_type}. Fix one of those so they " + "match!" + ) + CONFIG_MAPPING.register(model_type, config, exist_ok=exist_ok) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_bert.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..ea29fb81c435aa710dfb8912bf83c7068ed30ed6 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_bert.py @@ -0,0 +1,154 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""BERT model configuration""" + +from collections import OrderedDict +from typing import Mapping + +from ...configuration_utils import PretrainedConfig +from ...onnx import OnnxConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +class BertConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`BertModel`] or a [`TFBertModel`]. It is used to + instantiate a BERT model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the BERT + [google-bert/bert-base-uncased](https://huggingface.co/google-bert/bert-base-uncased) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 30522): + Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`BertModel`] or [`TFBertModel`]. + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (`int`, *optional*, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (`int`, *optional*, defaults to 2): + The vocabulary size of the `token_type_ids` passed when calling [`BertModel`] or [`TFBertModel`]. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the layer normalization layers. + position_embedding_type (`str`, *optional*, defaults to `"absolute"`): + Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For + positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to + [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). + For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models + with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658). + is_decoder (`bool`, *optional*, defaults to `False`): + Whether the model is used as a decoder or not. If `False`, the model is used as an encoder. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + classifier_dropout (`float`, *optional*): + The dropout ratio for the classification head. + + Examples: + + ```python + >>> from transformers import BertConfig, BertModel + + >>> # Initializing a BERT google-bert/bert-base-uncased style configuration + >>> configuration = BertConfig() + + >>> # Initializing a model (with random weights) from the google-bert/bert-base-uncased style configuration + >>> model = BertModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "bert" + + def __init__( + self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=0, + position_embedding_type="absolute", + use_cache=True, + classifier_dropout=None, + **kwargs, + ): + super().__init__(pad_token_id=pad_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.position_embedding_type = position_embedding_type + self.use_cache = use_cache + self.classifier_dropout = classifier_dropout + + +class BertOnnxConfig(OnnxConfig): + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + if self.task == "multiple-choice": + dynamic_axis = {0: "batch", 1: "choice", 2: "sequence"} + else: + dynamic_axis = {0: "batch", 1: "sequence"} + return OrderedDict( + [ + ("input_ids", dynamic_axis), + ("attention_mask", dynamic_axis), + ("token_type_ids", dynamic_axis), + ] + ) + + +__all__ = ["BertConfig", "BertOnnxConfig"] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_encoder_decoder.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_encoder_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..8b5c62363f6ad9c3ce72c61bd597dac3f9078c0a --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_encoder_decoder.py @@ -0,0 +1,111 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ...configuration_utils import PretrainedConfig +from ...utils import logging +from ..auto import AutoConfig + + +logger = logging.get_logger(__name__) + + +class EncoderDecoderConfig(PretrainedConfig): + r""" + [`EncoderDecoderConfig`] is the configuration class to store the configuration of a [`EncoderDecoderModel`]. It is + used to instantiate an Encoder Decoder model according to the specified arguments, defining the encoder and decoder + configs. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + kwargs (*optional*): + Dictionary of keyword arguments. Notably: + + - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines + the encoder config. + - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines + the decoder config. + + Examples: + + ```python + >>> from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel + + >>> # Initializing a BERT google-bert/bert-base-uncased style configuration + >>> config_encoder = BertConfig() + >>> config_decoder = BertConfig() + + >>> config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder) + + >>> # Initializing a Bert2Bert model (with random weights) from the google-bert/bert-base-uncased style configurations + >>> model = EncoderDecoderModel(config=config) + + >>> # Accessing the model configuration + >>> config_encoder = model.config.encoder + >>> config_decoder = model.config.decoder + >>> # set decoder config to causal lm + >>> config_decoder.is_decoder = True + >>> config_decoder.add_cross_attention = True + + >>> # Saving the model, including its configuration + >>> model.save_pretrained("my-model") + + >>> # loading model and config from pretrained folder + >>> encoder_decoder_config = EncoderDecoderConfig.from_pretrained("my-model") + >>> model = EncoderDecoderModel.from_pretrained("my-model", config=encoder_decoder_config) + ```""" + + model_type = "encoder-decoder" + sub_configs = {"encoder": AutoConfig, "decoder": AutoConfig} + is_composition = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if "encoder" not in kwargs or "decoder" not in kwargs: + raise ValueError( + f"A configuraton of type {self.model_type} cannot be instantiated because " + f"both `encoder` and `decoder` sub-configurations were not passed, only {kwargs}" + ) + encoder_config = kwargs.pop("encoder") + encoder_model_type = encoder_config.pop("model_type") + decoder_config = kwargs.pop("decoder") + decoder_model_type = decoder_config.pop("model_type") + + self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config) + self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config) + self.is_encoder_decoder = True + + @classmethod + def from_encoder_decoder_configs( + cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs + ) -> PretrainedConfig: + r""" + Instantiate a [`EncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model configuration and + decoder model configuration. + + Returns: + [`EncoderDecoderConfig`]: An instance of a configuration object + """ + logger.info("Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config") + decoder_config.is_decoder = True + decoder_config.add_cross_attention = True + + return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs) + + +__all__ = ["EncoderDecoderConfig"] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_qwen3.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_qwen3.py new file mode 100644 index 0000000000000000000000000000000000000000..06e527ce53f4b14086ed0343cc9879c536d96f2d --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_qwen3.py @@ -0,0 +1,212 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qwen3 model configuration""" + +from ...configuration_utils import PretrainedConfig +from ...modeling_rope_utils import rope_config_validation +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +class Qwen3Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Qwen3Model`]. It is used to instantiate a + Qwen3 model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of + Qwen3-8B [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 151936): + Vocabulary size of the Qwen3 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Qwen3Model`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 22016): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*, defaults to 32): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`. + head_dim (`int`, *optional*, defaults to 128): + The attention head dimension. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 32768): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (`float`, *optional*): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (`int`, *optional*): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (`float`, *optional*): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + use_sliding_window (`bool`, *optional*, defaults to `False`): + Whether to use sliding window attention. + sliding_window (`int`, *optional*, defaults to 4096): + Sliding window attention (SWA) window size. If not specified, will default to `4096`. + max_window_layers (`int`, *optional*, defaults to 28): + The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + + ```python + >>> from transformers import Qwen3Model, Qwen3Config + + >>> # Initializing a Qwen3 style configuration + >>> configuration = Qwen3Config() + + >>> # Initializing a model from the Qwen3-8B style configuration + >>> model = Qwen3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "qwen3" + keys_to_ignore_at_inference = ["past_key_values"] + + # Default tensor parallel plan for base model `Qwen3` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + def __init__( + self, + vocab_size=151936, + hidden_size=4096, + intermediate_size=22016, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=32, + head_dim=128, + hidden_act="silu", + max_position_embeddings=32768, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + use_sliding_window=False, + sliding_window=4096, + max_window_layers=28, + attention_dropout=0.0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.use_sliding_window = use_sliding_window + self.sliding_window = sliding_window # we check `use_sliding_window` in the modeling code + self.max_window_layers = max_window_layers + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + # Validate the correctness of rotary position embeddings parameters + # BC: if there is a 'type' field, move it to 'rope_type'. + if self.rope_scaling is not None and "type" in self.rope_scaling: + self.rope_scaling["rope_type"] = self.rope_scaling["type"] + rope_config_validation(self) + + super().__init__( + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +__all__ = ["Qwen3Config"] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_timm_backbone.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_timm_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..6000698c92488d046ef482eb664920e5bfc58ec5 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_timm_backbone.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configuration for Backbone models""" + +from ...configuration_utils import PretrainedConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +class TimmBackboneConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration for a timm backbone [`TimmBackbone`]. + + It is used to instantiate a timm backbone model according to the specified arguments, defining the model. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + backbone (`str`, *optional*): + The timm checkpoint to load. + num_channels (`int`, *optional*, defaults to 3): + The number of input channels. + features_only (`bool`, *optional*, defaults to `True`): + Whether to output only the features or also the logits. + use_pretrained_backbone (`bool`, *optional*, defaults to `True`): + Whether to use a pretrained backbone. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). Will default to the last stage if unset. + freeze_batch_norm_2d (`bool`, *optional*, defaults to `False`): + Converts all `BatchNorm2d` and `SyncBatchNorm` layers of provided module into `FrozenBatchNorm2d`. + + Example: + ```python + >>> from transformers import TimmBackboneConfig, TimmBackbone + + >>> # Initializing a timm backbone + >>> configuration = TimmBackboneConfig("resnet50") + + >>> # Initializing a model from the configuration + >>> model = TimmBackbone(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` + """ + + model_type = "timm_backbone" + + def __init__( + self, + backbone=None, + num_channels=3, + features_only=True, + use_pretrained_backbone=True, + out_indices=None, + freeze_batch_norm_2d=False, + **kwargs, + ): + super().__init__(**kwargs) + self.backbone = backbone + self.num_channels = num_channels + self.features_only = features_only + self.use_pretrained_backbone = use_pretrained_backbone + self.use_timm_backbone = True + self.out_indices = out_indices if out_indices is not None else [-1] + self.freeze_batch_norm_2d = freeze_batch_norm_2d + + +__all__ = ["TimmBackboneConfig"] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c743480d786b71a7dcf2e8f8896db69dba588851 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/configuration_utils.py @@ -0,0 +1,1632 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generation configuration class and utilities.""" + +import copy +import json +import os +import warnings +from abc import ABC, abstractmethod +from dataclasses import dataclass, is_dataclass +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +from .. import __version__ +from ..configuration_utils import PretrainedConfig +from ..utils import ( + GENERATION_CONFIG_NAME, + ExplicitEnum, + PushToHubMixin, + cached_file, + download_url, + extract_commit_hash, + is_remote_url, + is_torch_available, + logging, +) + + +if TYPE_CHECKING: + from ..modeling_utils import PreTrainedModel + + +logger = logging.get_logger(__name__) +METADATA_FIELDS = ("_from_model_config", "_commit_hash", "_original_object_hash", "transformers_version") +CACHE_CONFIG_MAPPING = {} +NEED_SETUP_CACHE_CLASSES_MAPPING = {} +QUANT_BACKEND_CLASSES_MAPPING = {} +ALL_CACHE_IMPLEMENTATIONS = [] + +if is_torch_available(): + from ..cache_utils import ( + HQQQuantizedCache, + HybridCache, + HybridChunkedCache, + MambaCache, + OffloadedStaticCache, + QuantizedCacheConfig, + QuantoQuantizedCache, + SlidingWindowCache, + StaticCache, + StaticCacheConfig, + ) + from .logits_process import SynthIDTextWatermarkLogitsProcessor, WatermarkLogitsProcessor + + CACHE_CONFIG_MAPPING["quantized"] = QuantizedCacheConfig + CACHE_CONFIG_MAPPING["static"] = StaticCacheConfig + NEED_SETUP_CACHE_CLASSES_MAPPING = { + "static": StaticCache, + "offloaded_static": OffloadedStaticCache, + "sliding_window": SlidingWindowCache, + "hybrid": HybridCache, + "hybrid_chunked": HybridChunkedCache, + "mamba": MambaCache, + } + QUANT_BACKEND_CLASSES_MAPPING = {"quanto": QuantoQuantizedCache, "HQQ": HQQQuantizedCache} + ALL_CACHE_IMPLEMENTATIONS = ( + list(NEED_SETUP_CACHE_CLASSES_MAPPING.keys()) + list(CACHE_CONFIG_MAPPING.keys()) + ["offloaded", "dynamic"] + ) + + +class GenerationMode(ExplicitEnum): + """ + Possible generation modes, downstream of the [`~generation.GenerationMixin.generate`] method. + """ + + # Non-beam methods + CONTRASTIVE_SEARCH = "contrastive_search" + GREEDY_SEARCH = "greedy_search" + SAMPLE = "sample" + ASSISTED_GENERATION = "assisted_generation" + DOLA_GENERATION = "dola_generation" + # Beam methods + BEAM_SEARCH = "beam_search" + BEAM_SAMPLE = "beam_sample" + CONSTRAINED_BEAM_SEARCH = "constrained_beam_search" + GROUP_BEAM_SEARCH = "group_beam_search" + + +class GenerationConfig(PushToHubMixin): + # no-format + """ + Class that holds a configuration for a generation task. A `generate` call supports the following generation methods + for text-decoder, text-to-text, speech-to-text, and vision-to-text models: + + - *greedy decoding* if `num_beams=1` and `do_sample=False` + - *contrastive search* if `penalty_alpha>0.` and `top_k>1` + - *multinomial sampling* if `num_beams=1` and `do_sample=True` + - *beam-search decoding* if `num_beams>1` and `do_sample=False` + - *beam-search multinomial sampling* if `num_beams>1` and `do_sample=True` + - *diverse beam-search decoding* if `num_beams>1` and `num_beam_groups>1` + - *constrained beam-search decoding* if `constraints!=None` or `force_words_ids!=None` + - *assisted decoding* if `assistant_model` or `prompt_lookup_num_tokens` is passed to `.generate()` + - *dola decoding* if `dola_layers` is passed to `.generate()` + + To learn more about decoding strategies refer to the [text generation strategies guide](../generation_strategies). + + + + A large number of these flags control the logits or the stopping criteria of the generation. Make sure you check + the [generate-related classes](https://huggingface.co/docs/transformers/internal/generation_utils) for a full + description of the possible manipulations, as well as examples of their usage. + + + + Arg: + > Parameters that control the length of the output + + max_length (`int`, *optional*, defaults to 20): + The maximum length the generated tokens can have. Corresponds to the length of the input prompt + + `max_new_tokens`. Its effect is overridden by `max_new_tokens`, if also set. + max_new_tokens (`int`, *optional*): + The maximum numbers of tokens to generate, ignoring the number of tokens in the prompt. + min_length (`int`, *optional*, defaults to 0): + The minimum length of the sequence to be generated. Corresponds to the length of the input prompt + + `min_new_tokens`. Its effect is overridden by `min_new_tokens`, if also set. + min_new_tokens (`int`, *optional*): + The minimum numbers of tokens to generate, ignoring the number of tokens in the prompt. + early_stopping (`bool` or `str`, *optional*, defaults to `False`): + Controls the stopping condition for beam-based methods, like beam-search. It accepts the following values: + `True`, where the generation stops as soon as there are `num_beams` complete candidates; `False`, where an + heuristic is applied and the generation stops when is it very unlikely to find better candidates; + `"never"`, where the beam search procedure only stops when there cannot be better candidates (canonical + beam search algorithm). + max_time (`float`, *optional*): + The maximum amount of time you allow the computation to run for in seconds. generation will still finish + the current pass after allocated time has been passed. + stop_strings (`str or List[str]`, *optional*): + A string or a list of strings that should terminate generation if the model outputs them. + + > Parameters that control the generation strategy used + + do_sample (`bool`, *optional*, defaults to `False`): + Whether or not to use sampling ; use greedy decoding otherwise. + num_beams (`int`, *optional*, defaults to 1): + Number of beams for beam search. 1 means no beam search. + num_beam_groups (`int`, *optional*, defaults to 1): + Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams. + [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details. + penalty_alpha (`float`, *optional*): + The values balance the model confidence and the degeneration penalty in contrastive search decoding. + dola_layers (`str` or `List[int]`, *optional*): + The layers to use for DoLa decoding. If `None`, DoLa decoding is not used. If a string, it must + be one of "low" or "high", which means using the lower part or higher part of the model layers, respectively. + "low" means the first half of the layers up to the first 20 layers, and "high" means the last half of the + layers up to the last 20 layers. + If a list of integers, it must contain the indices of the layers to use for candidate premature layers in DoLa. + The 0-th layer is the word embedding layer of the model. Set to `'low'` to improve long-answer reasoning tasks, + `'high'` to improve short-answer tasks. Check the [documentation](https://github.com/huggingface/transformers/blob/main/docs/source/en/generation_strategies.md) + or [the paper](https://arxiv.org/abs/2309.03883) for more details. + + > Parameters that control the cache + + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should use the past last key/values attentions (if applicable to the model) to + speed up decoding. + cache_implementation (`str`, *optional*, default to `None`): + Name of the cache class that will be instantiated in `generate`, for faster decoding. Possible values are: + + - `"dynamic"`: [`DynamicCache`] + - `"static"`: [`StaticCache`] + - `"offloaded_static"`: [`OffloadedStaticCache`] + - `"sliding_window"`: [`SlidingWindowCache`] + - `"hybrid"`: [`HybridCache`] + - `"mamba"`: [`MambaCache`] + - `"quantized"`: [`QuantizedCache`] + + If none is specified, we will use the default cache for the model (which is often [`DynamicCache`]). See + our [cache documentation](https://huggingface.co/docs/transformers/en/kv_cache) for further information. + cache_config (`CacheConfig` or `dict`, *optional*, default to `None`): + Arguments used in the key-value cache class can be passed in `cache_config`. Can be passed as a `Dict` and + it will be converted to its repsective `CacheConfig` internally. + Otherwise can be passed as a `CacheConfig` class matching the indicated `cache_implementation`. + return_legacy_cache (`bool`, *optional*, default to `True`): + Whether to return the legacy or new format of the cache when `DynamicCache` is used by default. + + > Parameters for manipulation of the model output logits + + temperature (`float`, *optional*, defaults to 1.0): + The value used to module the next token probabilities. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 1.0 + top_k (`int`, *optional*, defaults to 50): + The number of highest probability vocabulary tokens to keep for top-k-filtering. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 50. + top_p (`float`, *optional*, defaults to 1.0): + If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to + `top_p` or higher are kept for generation. This value is set in a model's `generation_config.json` file. If it isn't set, the default value is 1.0 + min_p (`float`, *optional*): + Minimum token probability, which will be scaled by the probability of the most likely token. It must be a + value between 0 and 1. Typical values are in the 0.01-0.2 range, comparably selective as setting `top_p` in + the 0.99-0.8 range (use the opposite of normal `top_p` values). + typical_p (`float`, *optional*, defaults to 1.0): + Local typicality measures how similar the conditional probability of predicting a target token next is to + the expected conditional probability of predicting a random token next, given the partial text already + generated. If set to float < 1, the smallest set of the most locally typical tokens with probabilities that + add up to `typical_p` or higher are kept for generation. See [this + paper](https://arxiv.org/pdf/2202.00666.pdf) for more details. + epsilon_cutoff (`float`, *optional*, defaults to 0.0): + If set to float strictly between 0 and 1, only tokens with a conditional probability greater than + `epsilon_cutoff` will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on the + size of the model. See [Truncation Sampling as Language Model + Desmoothing](https://arxiv.org/abs/2210.15191) for more details. + eta_cutoff (`float`, *optional*, defaults to 0.0): + Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly between + 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) * + exp(-entropy(softmax(next_token_logits)))`. The latter term is intuitively the expected next token + probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3, + depending on the size of the model. See [Truncation Sampling as Language Model + Desmoothing](https://arxiv.org/abs/2210.15191) for more details. + diversity_penalty (`float`, *optional*, defaults to 0.0): + This value is subtracted from a beam's score if it generates a token same as any beam from other group at a + particular time. Note that `diversity_penalty` is only effective if `group beam search` is enabled. + repetition_penalty (`float`, *optional*, defaults to 1.0): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + encoder_repetition_penalty (`float`, *optional*, defaults to 1.0): + The paramater for encoder_repetition_penalty. An exponential penalty on sequences that are not in the + original input. 1.0 means no penalty. + length_penalty (`float`, *optional*, defaults to 1.0): + Exponential penalty to the length that is used with beam-based generation. It is applied as an exponent to + the sequence length, which in turn is used to divide the score of the sequence. Since the score is the log + likelihood of the sequence (i.e. negative), `length_penalty` > 0.0 promotes longer sequences, while + `length_penalty` < 0.0 encourages shorter sequences. + no_repeat_ngram_size (`int`, *optional*, defaults to 0): + If set to int > 0, all ngrams of that size can only occur once. + bad_words_ids (`List[List[int]]`, *optional*): + List of list of token ids that are not allowed to be generated. Check + [`~generation.NoBadWordsLogitsProcessor`] for further documentation and examples. + force_words_ids (`List[List[int]]` or `List[List[List[int]]]`, *optional*): + List of token ids that must be generated. If given a `List[List[int]]`, this is treated as a simple list of + words that must be included, the opposite to `bad_words_ids`. If given `List[List[List[int]]]`, this + triggers a [disjunctive constraint](https://github.com/huggingface/transformers/issues/14081), where one + can allow different forms of each word. + renormalize_logits (`bool`, *optional*, defaults to `False`): + Whether to renormalize the logits after applying all the logits processors (including the custom + ones). It's highly recommended to set this flag to `True` as the search algorithms suppose the score logits + are normalized but some logit processors break the normalization. + constraints (`List[Constraint]`, *optional*): + Custom constraints that can be added to the generation to ensure that the output will contain the use of + certain tokens as defined by `Constraint` objects, in the most sensible way possible. + forced_bos_token_id (`int`, *optional*, defaults to `model.config.forced_bos_token_id`): + The id of the token to force as the first generated token after the `decoder_start_token_id`. Useful for + multilingual models like [mBART](../model_doc/mbart) where the first generated token needs to be the target + language token. + forced_eos_token_id (`int` or List[int]`, *optional*, defaults to `model.config.forced_eos_token_id`): + The id of the token to force as the last generated token when `max_length` is reached. Optionally, use a + list to set multiple *end-of-sequence* tokens. + remove_invalid_values (`bool`, *optional*, defaults to `model.config.remove_invalid_values`): + Whether to remove possible *nan* and *inf* outputs of the model to prevent the generation method to crash. + Note that using `remove_invalid_values` can slow down generation. + exponential_decay_length_penalty (`tuple(int, float)`, *optional*): + This Tuple adds an exponentially increasing length penalty, after a certain amount of tokens have been + generated. The tuple shall consist of: `(start_index, decay_factor)` where `start_index` indicates where + penalty starts and `decay_factor` represents the factor of exponential decay + suppress_tokens (`List[int]`, *optional*): + A list of tokens that will be suppressed at generation. The `SupressTokens` logit processor will set their + log probs to `-inf` so that they are not sampled. + begin_suppress_tokens (`List[int]`, *optional*): + A list of tokens that will be suppressed at the beginning of the generation. The `SupressBeginTokens` logit + processor will set their log probs to `-inf` so that they are not sampled. + forced_decoder_ids (`List[List[int]]`, *optional*): + A list of pairs of integers which indicates a mapping from generation indices to token indices that will be + forced before sampling. For example, `[[1, 123]]` means the second generated token will always be a token + of index 123. + sequence_bias (`Dict[Tuple[int], float]`, *optional*)): + Dictionary that maps a sequence of tokens to its bias term. Positive biases increase the odds of the + sequence being selected, while negative biases do the opposite. Check + [`~generation.SequenceBiasLogitsProcessor`] for further documentation and examples. + token_healing (`bool`, *optional*, defaults to `False`): + Heal tail tokens of prompts by replacing them with their appropriate extensions. + This enhances the quality of completions for prompts affected by greedy tokenization bias. + guidance_scale (`float`, *optional*): + The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`. + Higher guidance scale encourages the model to generate samples that are more closely linked to the input + prompt, usually at the expense of poorer quality. + low_memory (`bool`, *optional*): + Switch to sequential beam search and sequential topk for contrastive search to reduce peak memory. + Used with beam search and contrastive search. + watermarking_config (`BaseWatermarkingConfig` or `dict`, *optional*): + Arguments used to watermark the model outputs by adding a small bias to randomly selected set of "green" + tokens. See the docs of [`SynthIDTextWatermarkingConfig`] and [`WatermarkingConfig`] for more + details. If passed as `Dict`, it will be converted to a `WatermarkingConfig` internally. + + > Parameters that define the output variables of generate + + num_return_sequences (`int`, *optional*, defaults to 1): + The number of independently computed returned sequences for each element in the batch. + output_attentions (`bool`, *optional*, defaults to `False`): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more details. + output_hidden_states (`bool`, *optional*, defaults to `False`): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more details. + output_scores (`bool`, *optional*, defaults to `False`): + Whether or not to return the prediction scores. See `scores` under returned tensors for more details. + output_logits (`bool`, *optional*): + Whether or not to return the unprocessed prediction logit scores. See `logits` under returned tensors for + more details. + return_dict_in_generate (`bool`, *optional*, defaults to `False`): + Whether or not to return a [`~utils.ModelOutput`], as opposed to returning exclusively the generated + sequence. This flag must be set to `True` to return the generation cache (when `use_cache` is `True`) + or optional outputs (see flags starting with `output_`) + + > Special tokens that can be used at generation time + + pad_token_id (`int`, *optional*): + The id of the *padding* token. + bos_token_id (`int`, *optional*): + The id of the *beginning-of-sequence* token. + eos_token_id (`Union[int, List[int]]`, *optional*): + The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens. + + > Generation parameters exclusive to encoder-decoder models + + encoder_no_repeat_ngram_size (`int`, *optional*, defaults to 0): + If set to int > 0, all ngrams of that size that occur in the `encoder_input_ids` cannot occur in the + `decoder_input_ids`. + decoder_start_token_id (`int` or `List[int]`, *optional*): + If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token or a list of length + `batch_size`. Indicating a list enables different start ids for each element in the batch + (e.g. multilingual models with different target languages in one batch) + + > Generation parameters exclusive to assistant generation + is_assistant (`bool`, *optional*, defaults to `False`): + Whether the model is an assistant (draft) model. + num_assistant_tokens (`int`, *optional*, defaults to 20): + Defines the number of _speculative tokens_ that shall be generated by the assistant model before being + checked by the target model at each iteration. Higher values for `num_assistant_tokens` make the generation + more _speculative_ : If the assistant model is performant larger speed-ups can be reached, if the assistant + model requires lots of corrections, lower speed-ups are reached. + num_assistant_tokens_schedule (`str`, *optional*, defaults to `"constant"`): + Defines the schedule at which max assistant tokens shall be changed during inference. + - `"heuristic"`: When all speculative tokens are correct, increase `num_assistant_tokens` by 2 else + reduce by 1. `num_assistant_tokens` value is persistent over multiple generation calls with the same assistant model. + - `"heuristic_transient"`: Same as `"heuristic"` but `num_assistant_tokens` is reset to its initial value after each generation call. + - `"constant"`: `num_assistant_tokens` stays unchanged during generation + assistant_confidence_threshold (`float`, *optional*, defaults to 0.4): + The confidence threshold for the assistant model. If the assistant model's confidence in its prediction for the current token is lower + than this threshold, the assistant model stops the current token generation iteration, even if the number of _speculative tokens_ + (defined by `num_assistant_tokens`) is not yet reached. The assistant's confidence threshold is adjusted throughout the speculative iterations to reduce the number of unnecessary draft and target forward passes, biased towards avoiding false negatives. + `assistant_confidence_threshold` value is persistent over multiple generation calls with the same assistant model. + It is an unsupervised version of the dynamic speculation lookahead + from Dynamic Speculation Lookahead Accelerates Speculative Decoding of Large Language Models . + prompt_lookup_num_tokens (`int`, *optional*): + The number of tokens to be output as candidate tokens. + max_matching_ngram_size (`int`, *optional*): + The maximum ngram size to be considered for matching in the prompt. Default to 2 if not provided. + assistant_early_exit(`int`, *optional*): + If set to a positive integer, early exit of the model will be used as an assistant. Can only be used with + models that support early exit (i.e. models where logits from intermediate layers can be interpreted by the LM head). + assistant_lookbehind(`int`, *optional*, defaults to 10): + If set to a positive integer, the re-encodeing process will additionally consider the last `assistant_lookbehind` assistant tokens + to correctly align tokens. Can only be used with different tokenizers in speculative decoding. + See this [blog](https://huggingface.co/blog/universal_assisted_generation) for more details. + target_lookbehind(`int`, *optional*, defaults to 10): + If set to a positive integer, the re-encodeing process will additionally consider the last `target_lookbehind` target tokens + to correctly align tokens. Can only be used with different tokenizers in speculative decoding. + See this [blog](https://huggingface.co/blog/universal_assisted_generation) for more details. + + > Parameters related to performances and compilation + + compile_config (CompileConfig, *optional*): + If using a static cache, this controls how `generate` will `compile` the forward pass for performance + gains. + + disable_compile (`bool`, *optional*): Whether to disable the automatic compilation of the forward pass. Automatic compilation happens when specific criteria are met, including using a compileable cache. Please open an issue if you find the need to use this flag. + + > Wild card + + generation_kwargs: + Additional generation kwargs will be forwarded to the `generate` function of the model. Kwargs that are not + present in `generate`'s signature will be used in the model forward pass. + """ + + extra_output_flags = ("output_attentions", "output_hidden_states", "output_scores", "output_logits") + + def __init__(self, **kwargs): + # Parameters that control the length of the output + self.max_length = kwargs.pop("max_length", 20) + self.max_new_tokens = kwargs.pop("max_new_tokens", None) + self.min_length = kwargs.pop("min_length", 0) + self.min_new_tokens = kwargs.pop("min_new_tokens", None) + self.early_stopping = kwargs.pop("early_stopping", False) + self.max_time = kwargs.pop("max_time", None) + self.stop_strings = kwargs.pop("stop_strings", None) + + # Parameters that control the generation strategy used + self.do_sample = kwargs.pop("do_sample", False) + self.num_beams = kwargs.pop("num_beams", 1) + self.num_beam_groups = kwargs.pop("num_beam_groups", 1) + self.penalty_alpha = kwargs.pop("penalty_alpha", None) + self.dola_layers = kwargs.pop("dola_layers", None) + + # Parameters that control the cache + self.use_cache = kwargs.pop("use_cache", True) + self.cache_implementation = kwargs.pop("cache_implementation", None) + self.cache_config = kwargs.pop("cache_config", None) + if self.cache_implementation is not None and self.cache_implementation in CACHE_CONFIG_MAPPING: + cache_config_class = CACHE_CONFIG_MAPPING[self.cache_implementation] + if isinstance(self.cache_config, dict): + self.cache_config = cache_config_class.from_dict(self.cache_config) + self.return_legacy_cache = kwargs.pop("return_legacy_cache", None) + self.prefill_chunk_size = kwargs.pop("prefill_chunk_size", None) + + # Parameters for manipulation of the model output logits + self.temperature = kwargs.pop("temperature", 1.0) + self.top_k = kwargs.pop("top_k", 50) + self.top_p = kwargs.pop("top_p", 1.0) + self.min_p = kwargs.pop("min_p", None) + self.typical_p = kwargs.pop("typical_p", 1.0) + self.epsilon_cutoff = kwargs.pop("epsilon_cutoff", 0.0) + self.eta_cutoff = kwargs.pop("eta_cutoff", 0.0) + self.diversity_penalty = kwargs.pop("diversity_penalty", 0.0) + self.repetition_penalty = kwargs.pop("repetition_penalty", 1.0) + self.encoder_repetition_penalty = kwargs.pop("encoder_repetition_penalty", 1.0) + self.length_penalty = kwargs.pop("length_penalty", 1.0) + self.no_repeat_ngram_size = kwargs.pop("no_repeat_ngram_size", 0) + self.bad_words_ids = kwargs.pop("bad_words_ids", None) + self.force_words_ids = kwargs.pop("force_words_ids", None) + self.renormalize_logits = kwargs.pop("renormalize_logits", False) + self.constraints = kwargs.pop("constraints", None) + self.forced_bos_token_id = kwargs.pop("forced_bos_token_id", None) + self.forced_eos_token_id = kwargs.pop("forced_eos_token_id", None) + self.remove_invalid_values = kwargs.pop("remove_invalid_values", False) + self.exponential_decay_length_penalty = kwargs.pop("exponential_decay_length_penalty", None) + self.suppress_tokens = kwargs.pop("suppress_tokens", None) + self.begin_suppress_tokens = kwargs.pop("begin_suppress_tokens", None) + self.forced_decoder_ids = kwargs.pop("forced_decoder_ids", None) + self.sequence_bias = kwargs.pop("sequence_bias", None) + self.token_healing = kwargs.pop("token_healing", False) + self.guidance_scale = kwargs.pop("guidance_scale", None) + self.low_memory = kwargs.pop("low_memory", None) + watermarking_config = kwargs.pop("watermarking_config", None) + if watermarking_config is None: + self.watermarking_config = None + elif isinstance(watermarking_config, BaseWatermarkingConfig): + self.watermarking_config = watermarking_config + else: + self.watermarking_config = WatermarkingConfig.from_dict(watermarking_config) + + # Parameters that define the output variables of `generate` + self.num_return_sequences = kwargs.pop("num_return_sequences", 1) + self.output_attentions = kwargs.pop("output_attentions", False) + self.output_hidden_states = kwargs.pop("output_hidden_states", False) + self.output_scores = kwargs.pop("output_scores", False) + self.output_logits = kwargs.pop("output_logits", None) + self.return_dict_in_generate = kwargs.pop("return_dict_in_generate", False) + + # Special tokens that can be used at generation time + self.pad_token_id = kwargs.pop("pad_token_id", None) + self.bos_token_id = kwargs.pop("bos_token_id", None) + self.eos_token_id = kwargs.pop("eos_token_id", None) + + # Generation parameters exclusive to encoder-decoder models + self.encoder_no_repeat_ngram_size = kwargs.pop("encoder_no_repeat_ngram_size", 0) + self.decoder_start_token_id = kwargs.pop("decoder_start_token_id", None) + + # Assistant generation + self.is_assistant = False + self.num_assistant_tokens = kwargs.pop("num_assistant_tokens", 20) + self.num_assistant_tokens_schedule = kwargs.pop("num_assistant_tokens_schedule", "constant") + self.assistant_confidence_threshold = kwargs.pop("assistant_confidence_threshold", 0.4) + self.prompt_lookup_num_tokens = kwargs.pop("prompt_lookup_num_tokens", None) + self.max_matching_ngram_size = kwargs.pop("max_matching_ngram_size", None) + self.assistant_early_exit = kwargs.pop("assistant_early_exit", None) + ## assistant generation for different tokenizers, the windows size for assistant/target model + self.assistant_lookbehind = kwargs.pop("assistant_lookbehind", 10) + self.target_lookbehind = kwargs.pop("target_lookbehind", 10) + + # Performance + self.compile_config = kwargs.pop("compile_config", CompileConfig()) + self.disable_compile = kwargs.pop("disable_compile", False) + # Wild card + self.generation_kwargs = kwargs.pop("generation_kwargs", {}) + + # The remaining attributes do not parametrize `.generate()`, but are informative and/or used by the hub + # interface. + self._from_model_config = kwargs.pop("_from_model_config", False) + self._commit_hash = kwargs.pop("_commit_hash", None) + self.transformers_version = kwargs.pop("transformers_version", __version__) + + # Additional attributes without default values + if not self._from_model_config: + # we don't want to copy values from the model config if we're initializing a `GenerationConfig` from a + # model's default configuration file + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error(f"Can't set {key} with value {value} for {self}") + raise err + + # Validate the values of the attributes + self.validate(is_init=True) + + def __hash__(self): + return hash(self.to_json_string(ignore_metadata=True)) + + def __eq__(self, other): + if not isinstance(other, GenerationConfig): + return False + + self_without_metadata = self.to_json_string(use_diff=False, ignore_metadata=True) + other_without_metadata = other.to_json_string(use_diff=False, ignore_metadata=True) + return self_without_metadata == other_without_metadata + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string(ignore_metadata=True)}" + + def get_generation_mode(self, assistant_model: Optional["PreTrainedModel"] = None) -> GenerationMode: + """ + Returns the generation mode triggered by the [`GenerationConfig`] instance. + + Arg: + assistant_model (`PreTrainedModel`, *optional*): + The assistant model to be used for assisted generation. If set, the generation mode will be + assisted generation. + + Returns: + `GenerationMode`: The generation mode triggered by the instance. + """ + # TODO joao: find out a way of not depending on external fields (e.g. `assistant_model`), then make this a + # property and part of the `__repr__` + if self.constraints is not None or self.force_words_ids is not None: + generation_mode = GenerationMode.CONSTRAINED_BEAM_SEARCH + elif self.num_beams == 1: + if self.do_sample is False: + if ( + self.top_k is not None + and self.top_k > 1 + and self.penalty_alpha is not None + and self.penalty_alpha > 0 + ): + generation_mode = GenerationMode.CONTRASTIVE_SEARCH + else: + generation_mode = GenerationMode.GREEDY_SEARCH + else: + generation_mode = GenerationMode.SAMPLE + else: + if self.num_beam_groups > 1: + generation_mode = GenerationMode.GROUP_BEAM_SEARCH + elif self.do_sample is True: + generation_mode = GenerationMode.BEAM_SAMPLE + else: + generation_mode = GenerationMode.BEAM_SEARCH + + # Assisted generation may extend some generation modes + if ( + assistant_model is not None + or self.prompt_lookup_num_tokens is not None + or self.assistant_early_exit is not None + ): + if generation_mode in ("greedy_search", "sample"): + generation_mode = GenerationMode.ASSISTED_GENERATION + else: + raise ValueError( + "You've set `assistant_model`, which triggers assisted generate. Currently, assisted generate " + "is only supported with Greedy Search and Sample." + ) + + # DoLa generation may extend some generation modes + if self.dola_layers is not None: + if generation_mode in ("greedy_search", "sample"): + generation_mode = GenerationMode.DOLA_GENERATION + else: + raise ValueError( + "You've set `dola_layers`, which triggers DoLa generate. Currently, DoLa generate " + "is only supported with Greedy Search and Sample." + ) + return generation_mode + + def validate(self, is_init=False): + """ + Validates the values of the attributes of the [`GenerationConfig`] instance. Raises exceptions in the presence + of parameterization that can be detected as incorrect from the configuration instance alone. + + Note that some parameters not validated here are best validated at generate runtime, as they may depend on + other inputs and/or the model, such as parameters related to the generation length. + + Arg: + is_init (`bool`, *optional*, defaults to `False`): + Whether the validation is performed during the initialization of the instance. + """ + + # Validation of individual attributes + if self.early_stopping not in {True, False, "never"}: + raise ValueError(f"`early_stopping` must be a boolean or 'never', but is {self.early_stopping}.") + if self.max_new_tokens is not None and self.max_new_tokens <= 0: + raise ValueError(f"`max_new_tokens` must be greater than 0, but is {self.max_new_tokens}.") + if self.pad_token_id is not None and self.pad_token_id < 0: + warnings.warn( + f"`pad_token_id` should be positive but got {self.pad_token_id}. This will cause errors when batch " + "generating, if there is padding. Please set `pad_token_id` explicitly as " + "`model.generation_config.pad_token_id=PAD_TOKEN_ID` to avoid errors in generation" + ) + + # Validation of attribute relations: + fix_location = "" + if is_init: + fix_location = ( + " This was detected when initializing the generation config instance, which means the corresponding " + "file may hold incorrect parameterization and should be fixed." + ) + + # 1. detect sampling-only parameterization when not in sampling mode + if self.do_sample is False: + greedy_wrong_parameter_msg = ( + "`do_sample` is set to `False`. However, `{flag_name}` is set to `{flag_value}` -- this flag is only " + "used in sample-based generation modes. You should set `do_sample=True` or unset `{flag_name}`." + + fix_location + ) + if self.temperature is not None and self.temperature != 1.0: + warnings.warn( + greedy_wrong_parameter_msg.format(flag_name="temperature", flag_value=self.temperature), + UserWarning, + ) + if self.top_p is not None and self.top_p != 1.0: + warnings.warn( + greedy_wrong_parameter_msg.format(flag_name="top_p", flag_value=self.top_p), + UserWarning, + ) + if self.min_p is not None: + warnings.warn( + greedy_wrong_parameter_msg.format(flag_name="min_p", flag_value=self.min_p), + UserWarning, + ) + if self.typical_p is not None and self.typical_p != 1.0: + warnings.warn( + greedy_wrong_parameter_msg.format(flag_name="typical_p", flag_value=self.typical_p), + UserWarning, + ) + if ( + self.top_k is not None and self.top_k != 50 and self.penalty_alpha is None + ): # contrastive search uses top_k + warnings.warn( + greedy_wrong_parameter_msg.format(flag_name="top_k", flag_value=self.top_k), + UserWarning, + ) + if self.epsilon_cutoff is not None and self.epsilon_cutoff != 0.0: + warnings.warn( + greedy_wrong_parameter_msg.format(flag_name="epsilon_cutoff", flag_value=self.epsilon_cutoff), + UserWarning, + ) + if self.eta_cutoff is not None and self.eta_cutoff != 0.0: + warnings.warn( + greedy_wrong_parameter_msg.format(flag_name="eta_cutoff", flag_value=self.eta_cutoff), + UserWarning, + ) + + # 2. detect beam-only parameterization when not in beam mode + if self.num_beams is None: + warnings.warn("`num_beams` is set to None - defaulting to 1.", UserWarning) + self.num_beams = 1 + + if self.num_beams == 1: + single_beam_wrong_parameter_msg = ( + "`num_beams` is set to 1. However, `{flag_name}` is set to `{flag_value}` -- this flag is only used " + "in beam-based generation modes. You should set `num_beams>1` or unset `{flag_name}`." + fix_location + ) + if self.early_stopping is not False: + warnings.warn( + single_beam_wrong_parameter_msg.format(flag_name="early_stopping", flag_value=self.early_stopping), + UserWarning, + ) + if self.num_beam_groups is not None and self.num_beam_groups != 1: + warnings.warn( + single_beam_wrong_parameter_msg.format( + flag_name="num_beam_groups", flag_value=self.num_beam_groups + ), + UserWarning, + ) + if self.diversity_penalty is not None and self.diversity_penalty != 0.0: + warnings.warn( + single_beam_wrong_parameter_msg.format( + flag_name="diversity_penalty", flag_value=self.diversity_penalty + ), + UserWarning, + ) + if self.length_penalty is not None and self.length_penalty != 1.0: + warnings.warn( + single_beam_wrong_parameter_msg.format(flag_name="length_penalty", flag_value=self.length_penalty), + UserWarning, + ) + if self.constraints is not None: + warnings.warn( + single_beam_wrong_parameter_msg.format(flag_name="constraints", flag_value=self.constraints), + UserWarning, + ) + + # 3. detect incorrect paramaterization specific to advanced beam modes + else: + # constrained beam search + if self.constraints is not None or self.force_words_ids is not None: + constrained_wrong_parameter_msg = ( + "one of `constraints`, `force_words_ids` is not `None`, triggering constrained beam search. However, " + "`{flag_name}` is set to `{flag_value}`, which is incompatible with this generation mode. Set " + "`constraints` and `force_words_ids` to `None` or unset `{flag_name}` to continue." + fix_location + ) + if self.do_sample is True: + raise ValueError( + constrained_wrong_parameter_msg.format(flag_name="do_sample", flag_value=self.do_sample) + ) + if self.num_beam_groups is not None and self.num_beam_groups != 1: + raise ValueError( + constrained_wrong_parameter_msg.format( + flag_name="num_beam_groups", flag_value=self.num_beam_groups + ) + ) + # group beam search + if self.diversity_penalty != 0.0 or self.num_beam_groups != 1: + group_error_prefix = ( + "`diversity_penalty` is not 0.0 or `num_beam_groups` is not 1, triggering group beam search. In " + "this generation mode, " + ) + if self.do_sample is True: + raise ValueError(group_error_prefix + "`do_sample` must be set to `False`") + if self.num_beams % self.num_beam_groups != 0: + raise ValueError(group_error_prefix + "`num_beams` should be divisible by `num_beam_groups`") + if self.diversity_penalty == 0.0: + raise ValueError( + group_error_prefix + + "`diversity_penalty` should be greater than `0.0`, otherwise your groups will be identical." + ) + # DoLa generation + if self.dola_layers is not None and (self.repetition_penalty is None or self.repetition_penalty < 1.2): + warnings.warn( + "`dola_layers` is set to trigger DoLa decoding, but `repetition_penalty` is set to a value of " + f"{self.repetition_penalty}, which could induce unwanted repetition. The recommended value for " + "DoLa decoding is `repetition_penalty>=1.2`.", + UserWarning, + ) + + # 4. check `num_return_sequences` + if self.num_return_sequences != 1: + if self.num_beams == 1: + if self.do_sample is False: + raise ValueError( + "Greedy methods without beam search do not support `num_return_sequences` different than 1 " + f"(got {self.num_return_sequences})." + ) + elif self.num_return_sequences > self.num_beams: + raise ValueError( + f"`num_return_sequences` ({self.num_return_sequences}) has to be smaller or equal to `num_beams` " + f"({self.num_beams})." + ) + + # 5. check cache-related arguments + if self.cache_implementation is not None and self.cache_implementation not in ALL_CACHE_IMPLEMENTATIONS: + raise ValueError( + f"Invalid `cache_implementation` ({self.cache_implementation}). Choose one of: " + f"{ALL_CACHE_IMPLEMENTATIONS}" + ) + if self.cache_config is not None: + cache_class = CACHE_CONFIG_MAPPING.get(self.cache_implementation) + if cache_class is None: + raise ValueError( + "You provided a `cache_config` but the cache implementation you are using " + f"({self.cache_implementation}) does not require any config. Make sure to use the " + "correct cache implementation matching your cache config." + ) + if not isinstance(self.cache_config, cache_class): + self.cache_config = cache_class.from_dict(self.cache_config) + self.cache_config.validate() + if self.use_cache is False: + # In this case, all cache-related arguments should be unset. However, since `use_cache=False` is often used + # passed to `generate` directly to hot-fix cache issues, let's raise a warning instead of an error + # (otherwise a user might need to overwrite several parameters). + no_cache_warning = ( + "You have set `use_cache` to `False`, but {cache_arg} is set to {cache_arg_value}. {cache_arg} will " + "have no effect." + ) + for arg_name in ("cache_implementation", "cache_config", "return_legacy_cache"): + if getattr(self, arg_name) is not None: + logger.warning_once( + no_cache_warning.format(cache_arg=arg_name, cache_arg_value=getattr(self, arg_name)) + ) + + # 6. check watermarking arguments + if self.watermarking_config is not None: + if not ( + isinstance(self.watermarking_config, WatermarkingConfig) + or isinstance(self.watermarking_config, SynthIDTextWatermarkingConfig) + ): + warnings.warn( + "`watermarking_config` as a dict is deprecated. Please construct `watermarking_config` object with " + "`WatermarkingConfig` or `SynthIDTextWatermarkingConfig` class.", + FutureWarning, + ) + self.watermarking_config = WatermarkingConfig.from_dict(self.watermarking_config) + self.watermarking_config.validate() + + # 7. performances arguments + if not isinstance(self.compile_config, CompileConfig): + raise ValueError( + f"You provided `compile_config` as an instance of {type(self.compile_config)}, but it must be an instance of `CompileConfig`." + ) + + # 8. other incorrect combinations + if self.return_dict_in_generate is not True: + for extra_output_flag in self.extra_output_flags: + if getattr(self, extra_output_flag) is True: + warnings.warn( + f"`return_dict_in_generate` is NOT set to `True`, but `{extra_output_flag}` is. When " + f"`return_dict_in_generate` is not `True`, `{extra_output_flag}` is ignored.", + UserWarning, + ) + + # 8. check common issue: passing `generate` arguments inside the generation config + generate_arguments = ( + "logits_processor", + "stopping_criteria", + "prefix_allowed_tokens_fn", + "synced_gpus", + "assistant_model", + "streamer", + "negative_prompt_ids", + "negative_prompt_attention_mask", + ) + for arg in generate_arguments: + if hasattr(self, arg): + raise ValueError( + f"Argument `{arg}` is not a valid argument of `GenerationConfig`. It should be passed to " + "`generate()` (or a pipeline) directly." + ) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + config_file_name: Optional[Union[str, os.PathLike]] = None, + push_to_hub: bool = False, + **kwargs, + ): + r""" + Save a generation configuration object to the directory `save_directory`, so that it can be re-loaded using the + [`~GenerationConfig.from_pretrained`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the configuration JSON file will be saved (will be created if it does not exist). + config_file_name (`str` or `os.PathLike`, *optional*, defaults to `"generation_config.json"`): + Name of the generation configuration JSON file to be saved in `save_directory`. + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + + # At save time, validate the instance -- if any warning/exception is thrown, we refuse to save the instance. + # This strictness is enforced to prevent bad configurations from being saved and re-used. + try: + with warnings.catch_warnings(record=True) as caught_warnings: + self.validate() + if len(caught_warnings) > 0: + raise ValueError(str([w.message for w in caught_warnings])) + except ValueError as exc: + raise ValueError( + "The generation config instance is invalid -- `.validate()` throws warnings and/or exceptions. " + "Fix these issues to save the configuration.\n\nThrown during validation:\n" + str(exc) + ) + + use_auth_token = kwargs.pop("use_auth_token", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. " + "Please use `token` instead.", + FutureWarning, + ) + if kwargs.get("token", None) is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + kwargs["token"] = use_auth_token + + config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME + + if os.path.isfile(save_directory): + raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file") + + os.makedirs(save_directory, exist_ok=True) + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = self._create_repo(repo_id, **kwargs) + files_timestamps = self._get_files_timestamps(save_directory) + + output_config_file = os.path.join(save_directory, config_file_name) + + self.to_json_file(output_config_file, use_diff=True) + logger.info(f"Configuration saved in {output_config_file}") + + if push_to_hub: + self._upload_modified_files( + save_directory, + repo_id, + files_timestamps, + commit_message=commit_message, + token=kwargs.get("token"), + ) + + @classmethod + def from_pretrained( + cls, + pretrained_model_name: Union[str, os.PathLike], + config_file_name: Optional[Union[str, os.PathLike]] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ) -> "GenerationConfig": + r""" + Instantiate a [`GenerationConfig`] from a generation configuration file. + + Args: + pretrained_model_name (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a configuration file saved using the + [`~GenerationConfig.save_pretrained`] method, e.g., `./my_model_directory/`. + config_file_name (`str` or `os.PathLike`, *optional*, defaults to `"generation_config.json"`): + Name of the generation configuration JSON file to be loaded from `pretrained_model_name`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if + they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use + the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + + + + To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. + + + + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + If `False`, then this function returns just the final configuration object. + + If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs* is a + dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e., the + part of `kwargs` which has not been used to update `config` and is otherwise ignored. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can + specify the folder name here. + kwargs (`Dict[str, Any]`, *optional*): + The values in kwargs of any keys which are configuration attributes will be used to override the loaded + values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled + by the `return_unused_kwargs` keyword parameter. + + Returns: + [`GenerationConfig`]: The configuration object instantiated from this pretrained model. + + Examples: + + ```python + >>> from transformers import GenerationConfig + + >>> # Download configuration from huggingface.co and cache. + >>> generation_config = GenerationConfig.from_pretrained("openai-community/gpt2") + + >>> # E.g. config was saved using *save_pretrained('./test/saved_model/')* + >>> generation_config.save_pretrained("./test/saved_model/") + >>> generation_config = GenerationConfig.from_pretrained("./test/saved_model/") + + >>> # You can also specify configuration names to your generation configuration file + >>> generation_config.save_pretrained("./test/saved_model/", config_file_name="my_configuration.json") + >>> generation_config = GenerationConfig.from_pretrained("./test/saved_model/", "my_configuration.json") + + >>> # If you'd like to try a minor variation to an existing configuration, you can also pass generation + >>> # arguments to `.from_pretrained()`. Be mindful that typos and unused arguments will be ignored + >>> generation_config, unused_kwargs = GenerationConfig.from_pretrained( + ... "openai-community/gpt2", top_k=1, foo=False, do_sample=True, return_unused_kwargs=True + ... ) + >>> generation_config.top_k + 1 + + >>> unused_kwargs + {'foo': False} + ```""" + config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME + + resume_download = kwargs.pop("resume_download", None) + proxies = kwargs.pop("proxies", None) + use_auth_token = kwargs.pop("use_auth_token", None) + subfolder = kwargs.pop("subfolder", "") + from_pipeline = kwargs.pop("_from_pipeline", None) + from_auto_class = kwargs.pop("_from_auto", False) + commit_hash = kwargs.pop("_commit_hash", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + user_agent = {"file_type": "config", "from_auto_class": from_auto_class} + if from_pipeline is not None: + user_agent["using_pipeline"] = from_pipeline + + config_path = os.path.join(pretrained_model_name, config_file_name) + config_path = str(config_path) + + is_local = os.path.exists(config_path) + if os.path.isfile(os.path.join(subfolder, config_path)): + # Special case when config_path is a local file + resolved_config_file = config_path + is_local = True + elif is_remote_url(config_path): + configuration_file = config_path + resolved_config_file = download_url(config_path) + else: + configuration_file = config_file_name + try: + # Load from local folder or from cache or download from model Hub and cache + resolved_config_file = cached_file( + pretrained_model_name, + configuration_file, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + revision=revision, + subfolder=subfolder, + _commit_hash=commit_hash, + ) + commit_hash = extract_commit_hash(resolved_config_file, commit_hash) + except EnvironmentError: + # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to + # the original exception. + raise + except Exception: + # For any other exception, we throw a generic error. + raise EnvironmentError( + f"Can't load the configuration of '{pretrained_model_name}'. If you were trying to load it" + " from 'https://huggingface.co/models', make sure you don't have a local directory with the same" + f" name. Otherwise, make sure '{pretrained_model_name}' is the correct path to a directory" + f" containing a {configuration_file} file" + ) + + try: + # Load config dict + config_dict = cls._dict_from_json_file(resolved_config_file) + config_dict["_commit_hash"] = commit_hash + except (json.JSONDecodeError, UnicodeDecodeError): + raise EnvironmentError( + f"It looks like the config file at '{resolved_config_file}' is not a valid JSON file." + ) + + if is_local: + logger.info(f"loading configuration file {resolved_config_file}") + else: + logger.info(f"loading configuration file {configuration_file} from cache at {resolved_config_file}") + + if kwargs.get("return_unused_kwargs") is True: + config, unused_kwargs = cls.from_dict(config_dict, **kwargs) + config._original_object_hash = hash(config) # Hash to detect whether the instance was modified + return config, unused_kwargs + else: + config = cls.from_dict(config_dict, **kwargs) + config._original_object_hash = hash(config) # Hash to detect whether the instance was modified + return config + + @classmethod + def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]): + with open(json_file, "r", encoding="utf-8") as reader: + text = reader.read() + return json.loads(text) + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "GenerationConfig": + """ + Instantiates a [`GenerationConfig`] from a Python dictionary of parameters. + + Args: + config_dict (`Dict[str, Any]`): + Dictionary that will be used to instantiate the configuration object. + kwargs (`Dict[str, Any]`): + Additional parameters from which to initialize the configuration object. + + Returns: + [`GenerationConfig`]: The configuration object instantiated from those parameters. + """ + return_unused_kwargs = kwargs.pop("return_unused_kwargs", False) + # Those arguments may be passed along for our internal telemetry. + # We remove them so they don't appear in `return_unused_kwargs`. + kwargs.pop("_from_auto", None) + kwargs.pop("_from_pipeline", None) + # The commit hash might have been updated in the `config_dict`, we don't want the kwargs to erase that update. + if "_commit_hash" in kwargs and "_commit_hash" in config_dict: + kwargs["_commit_hash"] = config_dict["_commit_hash"] + + # The line below allows model-specific config to be loaded as well through kwargs, with safety checks. + # See https://github.com/huggingface/transformers/pull/21269 + config = cls(**{**config_dict, **kwargs}) + unused_kwargs = config.update(**kwargs) + + logger.info(f"Generate config {config}") + if return_unused_kwargs: + return config, unused_kwargs + else: + return config + + def dict_torch_dtype_to_str(self, d: Dict[str, Any]) -> None: + """ + Checks whether the passed dictionary and its nested dicts have a *torch_dtype* key and if it's not None, + converts torch.dtype to a string of just the type. For example, `torch.float32` get converted into *"float32"* + string, which can then be stored in the json format. + """ + if d.get("torch_dtype", None) is not None and not isinstance(d["torch_dtype"], str): + d["torch_dtype"] = str(d["torch_dtype"]).split(".")[1] + for value in d.values(): + if isinstance(value, dict): + self.dict_torch_dtype_to_str(value) + + def to_diff_dict(self) -> Dict[str, Any]: + """ + Removes all attributes from config which correspond to the default config attributes for better readability and + serializes to a Python dictionary. + + Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance, + """ + config_dict = self.to_dict() + + # get the default config dict + default_config_dict = GenerationConfig().to_dict() + + serializable_config_dict = {} + + # only serialize values that differ from the default config + for key, value in config_dict.items(): + if key not in default_config_dict or key == "transformers_version" or value != default_config_dict[key]: + serializable_config_dict[key] = value + + self.dict_torch_dtype_to_str(serializable_config_dict) + return serializable_config_dict + + def to_dict(self) -> Dict[str, Any]: + """ + Serializes this instance to a Python dictionary. + + Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + output = copy.deepcopy(self.__dict__) + + # Fields to ignore at serialization time + if "_commit_hash" in output: + del output["_commit_hash"] + if "_original_object_hash" in output: + del output["_original_object_hash"] + if "compile_config" in output: + del output["compile_config"] + + # Transformers version when serializing this file + output["transformers_version"] = __version__ + + self.dict_torch_dtype_to_str(output) + return output + + def to_json_string(self, use_diff: bool = True, ignore_metadata: bool = False) -> str: + """ + Serializes this instance to a JSON string. + + Args: + use_diff (`bool`, *optional*, defaults to `True`): + If set to `True`, only the difference between the config instance and the default `GenerationConfig()` + is serialized to JSON string. + ignore_metadata (`bool`, *optional*, defaults to `False`): + Whether to ignore the metadata fields present in the instance + + Returns: + `str`: String containing all the attributes that make up this configuration instance in JSON format. + """ + if use_diff is True: + config_dict = self.to_diff_dict() + else: + config_dict = self.to_dict() + + if ignore_metadata: + for metadata_field in METADATA_FIELDS: + config_dict.pop(metadata_field, None) + + def convert_keys_to_string(obj): + if isinstance(obj, dict): + return {str(key): convert_keys_to_string(value) for key, value in obj.items()} + elif isinstance(obj, list): + return [convert_keys_to_string(item) for item in obj] + else: + return obj + + def convert_dataclass_to_dict(obj): + if isinstance(obj, dict): + return {key: convert_dataclass_to_dict(value) for key, value in obj.items()} + elif is_dataclass(obj): + return obj.to_dict() + else: + return obj + + config_dict = convert_keys_to_string(config_dict) + config_dict = convert_dataclass_to_dict(config_dict) + + return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this configuration instance's parameters will be saved. + use_diff (`bool`, *optional*, defaults to `True`): + If set to `True`, only the difference between the config instance and the default `GenerationConfig()` + is serialized to JSON file. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + writer.write(self.to_json_string(use_diff=use_diff)) + + @classmethod + def from_model_config(cls, model_config: PretrainedConfig) -> "GenerationConfig": + """ + Instantiates a [`GenerationConfig`] from a [`PretrainedConfig`]. This function is useful to convert legacy + [`PretrainedConfig`] objects, which may contain generation parameters, into a stand-alone [`GenerationConfig`]. + + Args: + model_config (`PretrainedConfig`): + The model config that will be used to instantiate the generation config. + + Returns: + [`GenerationConfig`]: The configuration object instantiated from those parameters. + """ + config_dict = model_config.to_dict() + config_dict.pop("_from_model_config", None) + + # Removes all `None` from the model config dict -- this lets the generation config defaults to take hold + config_dict = {key: value for key, value in config_dict.items() if value is not None} + + generation_config = cls.from_dict(config_dict, return_unused_kwargs=False, _from_model_config=True) + + # Special case: some models have generation attributes set in the decoder. Use them if still unset in the + # generation config (which in turn is defined from the outer attributes of model config). + decoder_config = model_config.get_text_config(decoder=True) + if decoder_config is not model_config: + default_generation_config = GenerationConfig() + decoder_config_dict = decoder_config.to_dict() + for attr in generation_config.to_dict().keys(): + is_unset = getattr(generation_config, attr) == getattr(default_generation_config, attr) + if attr in decoder_config_dict and is_unset: + setattr(generation_config, attr, decoder_config_dict[attr]) + + # If any `output_...` flag is set to `True`, we ensure `return_dict_in_generate` is set to `True`. + if generation_config.return_dict_in_generate is False: + if any( + getattr(generation_config, extra_output_flag, False) + for extra_output_flag in generation_config.extra_output_flags + ): + generation_config.return_dict_in_generate = True + + # Hash to detect whether the instance was modified + generation_config._original_object_hash = hash(generation_config) + return generation_config + + def update(self, **kwargs): + """ + Updates attributes of this class instance with attributes from `kwargs` if they match existing attributes, + returning all the unused kwargs. + + Args: + kwargs (`Dict[str, Any]`): + Dictionary of attributes to tentatively update this class. + + Returns: + `Dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance. + """ + to_remove = [] + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + to_remove.append(key) + + # Confirm that the updated instance is still valid + self.validate() + + # Remove all the attributes that were updated, without modifying the input dict + unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove} + return unused_kwargs + + +@dataclass +class BaseWatermarkingConfig(ABC): + """Generic watermarking config""" + + @classmethod + def from_dict(cls, config_dict, **kwargs): + """ + Constructs a BaseWatermarkingConfig instance from a dictionary of parameters. + + Args: + config_dict (Dict[str, Any]): Dictionary containing configuration parameters. + **kwargs: Additional keyword arguments to override dictionary values. + + Returns: + BaseWatermarkingConfig: Instance of BaseWatermarkingConfig constructed from the dictionary. + """ + config = cls(**config_dict) + to_remove = [] + for key, value in kwargs.items(): + if hasattr(config, key): + setattr(config, key, value) + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + return config + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (Union[str, os.PathLike]): Path to the JSON file in which this configuration instance's parameters will be saved. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + config_dict = self.to_dict() + json_string = json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + writer.write(json_string) + + def to_dict(self) -> Dict[str, Any]: + """ + Serializes this instance to a Python dictionary. + + Returns: + Dict[str, Any]: Dictionary of all the attributes that make up this configuration instance. + """ + output = copy.deepcopy(self.__dict__) + return output + + def __iter__(self): + for attr, value in copy.deepcopy(self.__dict__).items(): + yield attr, value + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + def to_json_string(self): + """ + Serializes this instance to a JSON formatted string. + + Returns: + str: JSON formatted string representing the configuration instance. + """ + return json.dumps(self.__dict__, indent=2) + "\n" + + def update(self, **kwargs): + """ + Update the configuration attributes with new values. + + Args: + **kwargs: Keyword arguments representing configuration attributes and their new values. + """ + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + + @abstractmethod + def validate(self): ... + + @abstractmethod + def construct_processor(self, vocab_size): ... + + +@dataclass +class WatermarkingConfig(BaseWatermarkingConfig): + """ + Class that holds arguments for watermark generation and should be passed into `GenerationConfig` during `generate`. + See [this paper](https://arxiv.org/abs/2306.04634) for more details on the arguments. + + Accepts the following keys: + - greenlist_ratio (`float`): + Used for watermarking. The ratio of "green" tokens used to the vocabulary size. Defaults to 0.25. + - bias (`float`): + Used with watermarking. The bias added to the selected "green" tokens' logits. Defaults to 2.0. + - hashing_key (`int`): + Hashing key used for watermarking. Defaults to 15485863 (the millionth prime). + - seeding_scheme (`str`): + Algorithm to use for watermarking. Accepts values: + - "lefthash" (default): "green" tokens selection depend on the last token (Algorithm 2 from the paper) + - "selfhash": "green" tokens selection depends on the current token itself (Algorithm 3 from the paper) + The downside of this scheme is that it considers all possible next tokens and can be slower than "lefthash". + - context_width(`int`): + The context length of previous tokens to use in seeding. Higher context length makes watermarking more robust. + """ + + def __init__( + self, + greenlist_ratio: Optional[float] = 0.25, + bias: Optional[float] = 2.0, + hashing_key: Optional[int] = 15485863, + seeding_scheme: Optional[str] = "lefthash", + context_width: Optional[int] = 1, + ): + self.greenlist_ratio = greenlist_ratio + self.bias = bias + self.hashing_key = hashing_key + self.seeding_scheme = seeding_scheme + self.context_width = context_width + + def validate(self): + watermark_missing_arg_msg = ( + "Some of the keys in `watermarking_config` are defined incorrectly. `{key}` should be {correct_value}` " + "but found {found_value}" + ) + if self.seeding_scheme not in ["selfhash", "lefthash"]: + raise ValueError( + watermark_missing_arg_msg.format( + key="seeding_scheme", + correct_value="[`selfhash`, `lefthash`]", + found_value=self.seeding_scheme, + ), + ) + if not 0.0 <= self.greenlist_ratio <= 1.0: + raise ValueError( + watermark_missing_arg_msg.format( + key="greenlist_ratio", + correct_value="in range between 0.0 and 1.0", + found_value=self.seeding_scheme, + ), + ) + if not self.context_width >= 1: + raise ValueError( + watermark_missing_arg_msg.format( + key="context_width", + correct_value="a positive integer", + found_value=self.context_width, + ), + ) + + def construct_processor(self, vocab_size: int, device) -> "WatermarkLogitsProcessor": + return WatermarkLogitsProcessor( + vocab_size=vocab_size, + device=device, + greenlist_ratio=self.greenlist_ratio, + bias=self.bias, + hashing_key=self.hashing_key, + seeding_scheme=self.seeding_scheme, + context_width=self.context_width, + ) + + +@dataclass +class SynthIDTextWatermarkingConfig(BaseWatermarkingConfig): + """ + Class that holds arguments for watermark generation and should be passed into `GenerationConfig` during `generate`. + See [this paper](https://www.nature.com/articles/s41586-024-08025-4) for more details on the arguments. + + Args: + ngram_len (`int`): + Ngram length. + keys (`List[int]`): + A sequence of watermarking keys, one for each depth. + context_history_size (`int`, *optional*, defaults to 1024): + Size of the tensor to keep track of seen contexts. + sampling_table_seed (`int`, *optional*, defaults to 0): + Random seed to generate the sampling table. + sampling_table_size (`int`, *optional*, defaults to 65536): + Size of the sampling table. + skip_first_ngram_calls (`bool`, *optional*, defaults to `False`): + Whether to skip first ngram calls. + debug_mode (`bool`, optional, *optional*, defaults to `False`): + Logits are modified to uniform one got before watermarking modification is applied. This is to test the + implementation. + + Examples: + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer, SynthIDTextWatermarkingConfig + + >>> tokenizer = AutoTokenizer.from_pretrained('google/gemma-2-2b', padding_side="left") + >>> model = AutoModelForCausalLM.from_pretrained('google/gemma-2-2b') + + >>> # SynthID Text configuration + >>> watermarking_config = SynthIDTextWatermarkingConfig( + ... keys=[654, 400, 836, 123, 340, 443, 597, 160, 57], + ... ngram_len=5, + ... ) + + >>> # Generation with watermarking + >>> tokenized_prompts = tokenizer(["Once upon a time, "], return_tensors="pt", padding=True) + >>> output_sequences = model.generate( + ... **tokenized_prompts, watermarking_config=watermarking_config, do_sample=True, max_new_tokens=10 + ... ) + >>> watermarked_text = tokenizer.batch_decode(output_sequences, skip_special_tokens=True) + ``` + """ + + def __init__( + self, + ngram_len: int, + keys: List[int], + context_history_size: int = 1024, + sampling_table_seed: int = 0, + sampling_table_size: int = 2**16, + skip_first_ngram_calls: bool = False, + debug_mode: bool = False, + ): + self.ngram_len = ngram_len + self.keys = keys + self.sampling_table_size = sampling_table_size + self.sampling_table_seed = sampling_table_seed + self.context_history_size = context_history_size + self.skip_first_ngram_calls = skip_first_ngram_calls + self.debug_mode = debug_mode + + def validate(self): + watermark_missing_arg_msg = ( + "Some of the keys in `watermarking_config` are defined incorrectly. `{key}` should be {correct_value}` " + "but found {found_value}" + ) + if self.sampling_table_size > 2**24: + raise ValueError( + watermark_missing_arg_msg.format( + key="sampling_table_size", + correct_value="< 2**24", + found_value=self.sampling_table_size, + ), + ) + + def construct_processor(self, vocab_size: int, device) -> "WatermarkLogitsProcessor": + return SynthIDTextWatermarkLogitsProcessor( + ngram_len=self.ngram_len, + keys=self.keys, + sampling_table_size=self.sampling_table_size, + sampling_table_seed=self.sampling_table_seed, + context_history_size=self.context_history_size, + device=device, + skip_first_ngram_calls=self.skip_first_ngram_calls, + debug_mode=self.debug_mode, + ) + + +@dataclass +class CompileConfig: + """ + Class that holds arguments relative to `torch.compile` behavior, when using automatic compilation in `generate`. + See [`torch.compile`](https://pytorch.org/docs/stable/generated/torch.compile.html) for more details on the arguments. + + Args: + fullgraph (`bool`, *optional*, defaults to `True`): + If `True`, requires that the whole forward be capturable in a single graph. + dynamic (`bool` or `None`, *optional*): + Whether to try to use dynamic shape graphs. + backend (`str` or `Callable`, *optional*, defaults to `"inductor"`): + Backend to be used. + mode (`str`, *optional*, defaults to `"reduce-overhead"`): + Controls balance between performance and overhead. + options (`dict`, *optional*): + A dictionary of options to pass to the backend. + + Examples: + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer, CompileConfig + + >>> tokenizer = AutoTokenizer.from_pretrained('google/gemma-2-2b') + >>> model = AutoModelForCausalLM.from_pretrained('google/gemma-2-2b').cuda() + + >>> # Automatic compile configuration, used with static cache + >>> compile_config = CompileConfig(dynamic=True) + + >>> # Generation with static cache and compile config + >>> input = tokenizer.encode("Hello there, how", return_tensors="pt").cuda() + >>> output = model.generate( + ... input, do_sample=False, max_new_tokens=300, cache_implementation="static", compile_config=compile_config + ... ) + >>> output_text = tokenizer.batch_decode(output, skip_special_tokens=True)[0] + ``` + """ + + fullgraph: bool = True + dynamic: Optional[bool] = None + backend: Union[str, Callable] = "inductor" + mode: str = "reduce-overhead" + options: Optional[dict] = None + # Used to flag our `generate` call to compile on e.g. CPU. Often not optimal, but useful for testing purposes. + _compile_all_devices = None + + def to_dict(self) -> Dict[str, Any]: + """Serializes this instance to a Python dictionary.""" + return copy.deepcopy({key: value for key, value in self.__dict__.items() if key != "_compile_all_devices"}) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/constants.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..fefd1b4601da04e073ff2880099ccaf87d0b1666 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/constants.py @@ -0,0 +1,6 @@ +IMAGENET_DEFAULT_MEAN = [0.485, 0.456, 0.406] +IMAGENET_DEFAULT_STD = [0.229, 0.224, 0.225] +IMAGENET_STANDARD_MEAN = [0.5, 0.5, 0.5] +IMAGENET_STANDARD_STD = [0.5, 0.5, 0.5] +OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073] +OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/convert.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/convert.py new file mode 100644 index 0000000000000000000000000000000000000000..58bc51f8e801cfaeb9423342f330c269fa5c2318 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/convert.py @@ -0,0 +1,460 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from inspect import signature +from itertools import chain +from pathlib import Path +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union + +import numpy as np +from packaging.version import Version, parse + +from ..tokenization_utils_base import PreTrainedTokenizerBase +from ..utils import ( + TensorType, + is_tf_available, + is_torch_available, + logging, +) +from .config import OnnxConfig + + +if is_torch_available(): + from ..modeling_utils import PreTrainedModel + +if is_tf_available(): + from ..modeling_tf_utils import TFPreTrainedModel + +if TYPE_CHECKING: + from ..feature_extraction_utils import FeatureExtractionMixin + from ..processing_utils import ProcessorMixin + from ..tokenization_utils import PreTrainedTokenizer + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# This is the minimal required version to support some ONNX Runtime features +ORT_QUANTIZE_MINIMUM_VERSION = parse("1.4.0") + + +def check_onnxruntime_requirements(minimum_version: Version): + """ + Check onnxruntime is installed and if the installed version match is recent enough + + Raises: + ImportError: If onnxruntime is not installed or too old version is found + """ + try: + import onnxruntime + + # Parse the version of the installed onnxruntime + ort_version = parse(onnxruntime.__version__) + + # We require 1.4.0 minimum + if ort_version < ORT_QUANTIZE_MINIMUM_VERSION: + raise ImportError( + f"We found an older version of onnxruntime ({onnxruntime.__version__}) " + f"but we require onnxruntime to be >= {minimum_version} to enable all the conversions options.\n" + "Please update onnxruntime by running `pip install --upgrade onnxruntime`" + ) + + except ImportError: + raise ImportError( + "onnxruntime doesn't seem to be currently installed. " + "Please install the onnxruntime by running `pip install onnxruntime`" + " and relaunch the conversion." + ) + + +def export_pytorch( + preprocessor: Union["PreTrainedTokenizer", "FeatureExtractionMixin", "ProcessorMixin"], + model: "PreTrainedModel", + config: OnnxConfig, + opset: int, + output: Path, + tokenizer: Optional["PreTrainedTokenizer"] = None, + device: str = "cpu", +) -> Tuple[List[str], List[str]]: + """ + Export a PyTorch model to an ONNX Intermediate Representation (IR) + + Args: + preprocessor: ([`PreTrainedTokenizer`], [`FeatureExtractionMixin`] or [`ProcessorMixin`]): + The preprocessor used for encoding the data. + model ([`PreTrainedModel`]): + The model to export. + config ([`~onnx.config.OnnxConfig`]): + The ONNX configuration associated with the exported model. + opset (`int`): + The version of the ONNX operator set to use. + output (`Path`): + Directory to store the exported ONNX model. + device (`str`, *optional*, defaults to `cpu`): + The device on which the ONNX model will be exported. Either `cpu` or `cuda`. + + Returns: + `Tuple[List[str], List[str]]`: A tuple with an ordered list of the model's inputs, and the named inputs from + the ONNX configuration. + """ + + if isinstance(preprocessor, PreTrainedTokenizerBase) and tokenizer is not None: + raise ValueError("You cannot provide both a tokenizer and a preprocessor to export the model.") + if tokenizer is not None: + warnings.warn( + "The `tokenizer` argument is deprecated and will be removed in version 5 of Transformers. Use" + " `preprocessor` instead.", + FutureWarning, + ) + logger.info("Overwriting the `preprocessor` argument with `tokenizer` to generate dummy inputs.") + preprocessor = tokenizer + + if issubclass(type(model), PreTrainedModel): + import torch + from torch.onnx import export as onnx_export + + logger.info(f"Using framework PyTorch: {torch.__version__}") + with torch.no_grad(): + model.config.return_dict = True + model.eval() + + # Check if we need to override certain configuration item + if config.values_override is not None: + logger.info(f"Overriding {len(config.values_override)} configuration item(s)") + for override_config_key, override_config_value in config.values_override.items(): + logger.info(f"\t- {override_config_key} -> {override_config_value}") + setattr(model.config, override_config_key, override_config_value) + + # Ensure inputs match + # TODO: Check when exporting QA we provide "is_pair=True" + model_inputs = config.generate_dummy_inputs(preprocessor, framework=TensorType.PYTORCH) + device = torch.device(device) + if device.type == "cuda" and torch.cuda.is_available(): + model.to(device) + model_inputs_device = {} + for k, v in model_inputs.items(): + if isinstance(v, Tuple): + model_inputs_device[k] = tuple( + x.to(device) if isinstance(x, torch.Tensor) else None for x in v + ) + elif isinstance(v, List): + model_inputs_device[k] = [ + tuple(x.to(device) if isinstance(x, torch.Tensor) else None for x in t) for t in v + ] + else: + model_inputs_device[k] = v.to(device) + + model_inputs = model_inputs_device + + inputs_match, matched_inputs = ensure_model_and_config_inputs_match(model, model_inputs.keys()) + onnx_outputs = list(config.outputs.keys()) + + if not inputs_match: + raise ValueError("Model and config inputs doesn't match") + + config.patch_ops() + + onnx_export( + model, + (model_inputs,), + f=output.as_posix(), + input_names=list(config.inputs.keys()), + output_names=onnx_outputs, + dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), + do_constant_folding=True, + opset_version=opset, + ) + + config.restore_ops() + + return matched_inputs, onnx_outputs + + +def export_tensorflow( + preprocessor: Union["PreTrainedTokenizer", "FeatureExtractionMixin"], + model: "TFPreTrainedModel", + config: OnnxConfig, + opset: int, + output: Path, + tokenizer: Optional["PreTrainedTokenizer"] = None, +) -> Tuple[List[str], List[str]]: + """ + Export a TensorFlow model to an ONNX Intermediate Representation (IR) + + Args: + preprocessor: ([`PreTrainedTokenizer`] or [`FeatureExtractionMixin`]): + The preprocessor used for encoding the data. + model ([`TFPreTrainedModel`]): + The model to export. + config ([`~onnx.config.OnnxConfig`]): + The ONNX configuration associated with the exported model. + opset (`int`): + The version of the ONNX operator set to use. + output (`Path`): + Directory to store the exported ONNX model. + + Returns: + `Tuple[List[str], List[str]]`: A tuple with an ordered list of the model's inputs, and the named inputs from + the ONNX configuration. + """ + import onnx + import tensorflow as tf + import tf2onnx + + if isinstance(preprocessor, PreTrainedTokenizerBase) and tokenizer is not None: + raise ValueError("You cannot provide both a tokenizer and preprocessor to export the model.") + if tokenizer is not None: + warnings.warn( + "The `tokenizer` argument is deprecated and will be removed in version 5 of Transformers. Use" + " `preprocessor` instead.", + FutureWarning, + ) + logger.info("Overwriting the `preprocessor` argument with `tokenizer` to generate dummy inputs.") + preprocessor = tokenizer + + model.config.return_dict = True + + # Check if we need to override certain configuration item + if config.values_override is not None: + logger.info(f"Overriding {len(config.values_override)} configuration item(s)") + for override_config_key, override_config_value in config.values_override.items(): + logger.info(f"\t- {override_config_key} -> {override_config_value}") + setattr(model.config, override_config_key, override_config_value) + + # Ensure inputs match + model_inputs = config.generate_dummy_inputs(preprocessor, framework=TensorType.TENSORFLOW) + inputs_match, matched_inputs = ensure_model_and_config_inputs_match(model, model_inputs.keys()) + onnx_outputs = list(config.outputs.keys()) + + input_signature = [ + tf.TensorSpec([None] * tensor.ndim, dtype=tensor.dtype, name=key) for key, tensor in model_inputs.items() + ] + onnx_model, _ = tf2onnx.convert.from_keras(model, input_signature, opset=opset) + onnx.save(onnx_model, output.as_posix()) + config.restore_ops() + + return matched_inputs, onnx_outputs + + +def export( + preprocessor: Union["PreTrainedTokenizer", "FeatureExtractionMixin", "ProcessorMixin"], + model: Union["PreTrainedModel", "TFPreTrainedModel"], + config: OnnxConfig, + opset: int, + output: Path, + tokenizer: Optional["PreTrainedTokenizer"] = None, + device: str = "cpu", +) -> Tuple[List[str], List[str]]: + """ + Export a Pytorch or TensorFlow model to an ONNX Intermediate Representation (IR) + + Args: + preprocessor: ([`PreTrainedTokenizer`], [`FeatureExtractionMixin`] or [`ProcessorMixin`]): + The preprocessor used for encoding the data. + model ([`PreTrainedModel`] or [`TFPreTrainedModel`]): + The model to export. + config ([`~onnx.config.OnnxConfig`]): + The ONNX configuration associated with the exported model. + opset (`int`): + The version of the ONNX operator set to use. + output (`Path`): + Directory to store the exported ONNX model. + device (`str`, *optional*, defaults to `cpu`): + The device on which the ONNX model will be exported. Either `cpu` or `cuda`. Only PyTorch is supported for + export on CUDA devices. + + Returns: + `Tuple[List[str], List[str]]`: A tuple with an ordered list of the model's inputs, and the named inputs from + the ONNX configuration. + """ + if not (is_torch_available() or is_tf_available()): + raise ImportError( + "Cannot convert because neither PyTorch nor TensorFlow are not installed. " + "Please install torch or tensorflow first." + ) + + if is_tf_available() and isinstance(model, TFPreTrainedModel) and device == "cuda": + raise RuntimeError("`tf2onnx` does not support export on CUDA device.") + + if isinstance(preprocessor, PreTrainedTokenizerBase) and tokenizer is not None: + raise ValueError("You cannot provide both a tokenizer and a preprocessor to export the model.") + if tokenizer is not None: + warnings.warn( + "The `tokenizer` argument is deprecated and will be removed in version 5 of Transformers. Use" + " `preprocessor` instead.", + FutureWarning, + ) + logger.info("Overwriting the `preprocessor` argument with `tokenizer` to generate dummy inputs.") + preprocessor = tokenizer + + if is_torch_available(): + from ..utils import get_torch_version + + if not config.is_torch_support_available: + logger.warning( + f"Unsupported PyTorch version for this model. Minimum required is {config.torch_onnx_minimum_version}," + f" got: {get_torch_version()}" + ) + + if is_torch_available() and issubclass(type(model), PreTrainedModel): + return export_pytorch(preprocessor, model, config, opset, output, tokenizer=tokenizer, device=device) + elif is_tf_available() and issubclass(type(model), TFPreTrainedModel): + return export_tensorflow(preprocessor, model, config, opset, output, tokenizer=tokenizer) + + +def validate_model_outputs( + config: OnnxConfig, + preprocessor: Union["PreTrainedTokenizer", "FeatureExtractionMixin", "ProcessorMixin"], + reference_model: Union["PreTrainedModel", "TFPreTrainedModel"], + onnx_model: Path, + onnx_named_outputs: List[str], + atol: float, + tokenizer: Optional["PreTrainedTokenizer"] = None, +): + from onnxruntime import InferenceSession, SessionOptions + + logger.info("Validating ONNX model...") + + if isinstance(preprocessor, PreTrainedTokenizerBase) and tokenizer is not None: + raise ValueError("You cannot provide both a tokenizer and a preprocessor to validate the model outputs.") + if tokenizer is not None: + warnings.warn( + "The `tokenizer` argument is deprecated and will be removed in version 5 of Transformers. Use" + " `preprocessor` instead.", + FutureWarning, + ) + logger.info("Overwriting the `preprocessor` argument with `tokenizer` to generate dummy inputs.") + preprocessor = tokenizer + + # generate inputs with a different batch_size and seq_len that was used for conversion to properly test + # dynamic input shapes. + if is_torch_available() and issubclass(type(reference_model), PreTrainedModel): + reference_model_inputs = config.generate_dummy_inputs( + preprocessor, + batch_size=config.default_fixed_batch + 1, + seq_length=config.default_fixed_sequence + 1, + framework=TensorType.PYTORCH, + ) + else: + reference_model_inputs = config.generate_dummy_inputs( + preprocessor, + batch_size=config.default_fixed_batch + 1, + seq_length=config.default_fixed_sequence + 1, + framework=TensorType.TENSORFLOW, + ) + + # Create ONNX Runtime session + options = SessionOptions() + session = InferenceSession(onnx_model.as_posix(), options, providers=["CPUExecutionProvider"]) + + # Compute outputs from the reference model + if is_torch_available() and issubclass(type(reference_model), PreTrainedModel): + reference_model.to("cpu") + ref_outputs = reference_model(**reference_model_inputs) + ref_outputs_dict = {} + + # We flatten potential collection of outputs (i.e. past_keys) to a flat structure + for name, value in ref_outputs.items(): + # Overwriting the output name as "present" since it is the name used for the ONNX outputs + # ("past_key_values" being taken for the ONNX inputs) + if name == "past_key_values": + name = "present" + if isinstance(value, (list, tuple)): + value = config.flatten_output_collection_property(name, value) + ref_outputs_dict.update(value) + else: + ref_outputs_dict[name] = value + + # Create onnxruntime inputs from the reference model inputs + reference_model_inputs_onnxruntime = config.generate_dummy_inputs_onnxruntime(reference_model_inputs) + + # We flatten potential collection of inputs (i.e. past_keys) + onnx_inputs = {} + for name, value in reference_model_inputs_onnxruntime.items(): + if isinstance(value, (list, tuple)): + value = config.flatten_output_collection_property(name, value) + onnx_inputs.update({tensor_name: pt_tensor.numpy() for tensor_name, pt_tensor in value.items()}) + else: + onnx_inputs[name] = value.numpy() + + # Compute outputs from the ONNX model + onnx_outputs = session.run(onnx_named_outputs, onnx_inputs) + + # Check we have a subset of the keys into onnx_outputs against ref_outputs + ref_outputs_set, onnx_outputs_set = set(ref_outputs_dict.keys()), set(onnx_named_outputs) + if not onnx_outputs_set.issubset(ref_outputs_set): + logger.info( + f"\t-[x] ONNX model output names {onnx_outputs_set} do not match reference model {ref_outputs_set}" + ) + + raise ValueError( + "Outputs doesn't match between reference model and ONNX exported model: " + f"{onnx_outputs_set.difference(ref_outputs_set)}" + ) + else: + logger.info(f"\t-[✓] ONNX model output names match reference model ({onnx_outputs_set})") + + # Check the shape and values match + for name, ort_value in zip(onnx_named_outputs, onnx_outputs): + if is_torch_available() and issubclass(type(reference_model), PreTrainedModel): + ref_value = ref_outputs_dict[name].detach().numpy() + else: + ref_value = ref_outputs_dict[name].numpy() + logger.info(f'\t- Validating ONNX Model output "{name}":') + + # Shape + if not ort_value.shape == ref_value.shape: + logger.info(f"\t\t-[x] shape {ort_value.shape} doesn't match {ref_value.shape}") + raise ValueError( + "Outputs shape doesn't match between reference model and ONNX exported model: " + f"Got {ref_value.shape} (reference) and {ort_value.shape} (ONNX)" + ) + else: + logger.info(f"\t\t-[✓] {ort_value.shape} matches {ref_value.shape}") + + # Values + if not np.allclose(ref_value, ort_value, atol=atol): + bad_indices = np.logical_not(np.isclose(ref_value, ort_value, atol=atol)) + logger.info(f"\t\t-[x] values not close enough (atol: {atol})") + raise ValueError( + "Outputs values doesn't match between reference model and ONNX exported model: " + f"Got max absolute difference of: {np.amax(np.abs(ref_value - ort_value))} for " + f"{ref_value[bad_indices]} vs {ort_value[bad_indices]}" + ) + else: + logger.info(f"\t\t-[✓] all values close (atol: {atol})") + + +def ensure_model_and_config_inputs_match( + model: Union["PreTrainedModel", "TFPreTrainedModel"], model_inputs: Iterable[str] +) -> Tuple[bool, List[str]]: + """ + + :param model_inputs: :param config_inputs: :return: + """ + if is_torch_available() and issubclass(type(model), PreTrainedModel): + forward_parameters = signature(model.forward).parameters + else: + forward_parameters = signature(model.call).parameters + model_inputs_set = set(model_inputs) + + # We are fine if config_inputs has more keys than model_inputs + forward_inputs_set = set(forward_parameters.keys()) + is_ok = model_inputs_set.issubset(forward_inputs_set) + + # Make sure the input order match (VERY IMPORTANT !!!!) + matching_inputs = forward_inputs_set.intersection(model_inputs_set) + ordered_inputs = [parameter for parameter in forward_parameters.keys() if parameter in matching_inputs] + return is_ok, ordered_inputs diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/convert_slow_tokenizer.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/convert_slow_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..c8cc1cdbe97ba150f62230eea53b67ab80dd1edc --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/convert_slow_tokenizer.py @@ -0,0 +1,1736 @@ +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Utilities to convert slow tokenizers in their fast tokenizers counterparts. + +All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and +allow to make our dependency on SentencePiece optional. +""" + +import warnings + +from packaging import version +from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors +from tokenizers.models import BPE, Unigram, WordPiece + +from .utils import is_protobuf_available, is_sentencepiece_available, logging, requires_backends +from .utils.import_utils import PROTOBUF_IMPORT_ERROR + + +logger = logging.get_logger(__name__) + + +def import_protobuf(error_message=""): + if is_sentencepiece_available(): + from sentencepiece import sentencepiece_model_pb2 + + return sentencepiece_model_pb2 + if is_protobuf_available(): + import google.protobuf + + if version.parse(google.protobuf.__version__) < version.parse("4.0.0"): + from transformers.utils import sentencepiece_model_pb2 + else: + from transformers.utils import sentencepiece_model_pb2_new as sentencepiece_model_pb2 + return sentencepiece_model_pb2 + else: + raise ImportError(PROTOBUF_IMPORT_ERROR.format(error_message)) + + +def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str: + if add_prefix_space: + prepend_scheme = "always" + if not getattr(original_tokenizer, "legacy", True): + prepend_scheme = "first" + else: + prepend_scheme = "never" + return prepend_scheme + + +def generate_merges(vocab, vocab_scores): + reverse = vocab_scores is not None + vocab_scores = dict(vocab_scores) if reverse else vocab + + merges = [] + for merge, piece_score in vocab_scores.items(): + local = [] + for index in range(1, len(merge)): + piece_l, piece_r = merge[:index], merge[index:] + if piece_l in vocab and piece_r in vocab: + local.append((piece_l, piece_r, piece_score)) + local = sorted(local, key=lambda x: (vocab[x[0]], vocab[x[1]])) + merges.extend(local) + + merges = sorted(merges, key=lambda val: (val[2], len(val[0]), len(val[1])), reverse=reverse) + merges = [(val[0], val[1]) for val in merges] + return merges + + +class SentencePieceExtractor: + """ + Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece + """ + + def __init__(self, model: str): + requires_backends(self, "sentencepiece") + from sentencepiece import SentencePieceProcessor + + self.sp = SentencePieceProcessor() + self.sp.Load(model) + + def extract(self, vocab_scores=None) -> tuple[dict[str, int], list[tuple]]: + """ + By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to + order the merges with respect to the piece scores instead. + """ + sp = self.sp + vocab = {sp.id_to_piece(index): index for index in range(sp.GetPieceSize())} + + merges = generate_merges(vocab, vocab_scores) + + return vocab, merges + + +class GemmaSentencePieceExtractor(SentencePieceExtractor): + def extract(self, vocab_scores=None) -> tuple[dict[str, int], list[tuple]]: + """ + By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to + order the merges with respect to the piece scores instead. + """ + sp = self.sp + vocab = {sp.id_to_piece(index): index for index in range(sp.GetPieceSize())} + + # If "\t" is missing in the vocab, we have to do this to support merges + # "<0x09>" is the bytefallback for `\t` + if "\t" not in vocab: + vocab["\t"] = vocab.get("<0x09>") + merges = generate_merges(vocab, vocab_scores) + return vocab, merges + + +def check_number_comma(piece: str) -> bool: + return len(piece) < 2 or piece[-1] != "," or not piece[-2].isdigit() + + +class Converter: + def __init__(self, original_tokenizer): + self.original_tokenizer = original_tokenizer + + def converted(self) -> Tokenizer: + raise NotImplementedError() + + +class BertConverter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.vocab + tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token))) + + tokenize_chinese_chars = False + strip_accents = False + do_lower_case = False + if hasattr(self.original_tokenizer, "basic_tokenizer"): + tokenize_chinese_chars = self.original_tokenizer.basic_tokenizer.tokenize_chinese_chars + strip_accents = self.original_tokenizer.basic_tokenizer.strip_accents + do_lower_case = self.original_tokenizer.basic_tokenizer.do_lower_case + + tokenizer.normalizer = normalizers.BertNormalizer( + clean_text=True, + handle_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + lowercase=do_lower_case, + ) + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + + cls = str(self.original_tokenizer.cls_token) + sep = str(self.original_tokenizer.sep_token) + cls_token_id = self.original_tokenizer.cls_token_id + sep_token_id = self.original_tokenizer.sep_token_id + + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{cls}:0 $A:0 {sep}:0", + pair=f"{cls}:0 $A:0 {sep}:0 $B:1 {sep}:1", + special_tokens=[ + (cls, cls_token_id), + (sep, sep_token_id), + ], + ) + tokenizer.decoder = decoders.WordPiece(prefix="##") + + return tokenizer + + +class SplinterConverter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.vocab + tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token))) + + tokenize_chinese_chars = False + strip_accents = False + do_lower_case = False + if hasattr(self.original_tokenizer, "basic_tokenizer"): + tokenize_chinese_chars = self.original_tokenizer.basic_tokenizer.tokenize_chinese_chars + strip_accents = self.original_tokenizer.basic_tokenizer.strip_accents + do_lower_case = self.original_tokenizer.basic_tokenizer.do_lower_case + + tokenizer.normalizer = normalizers.BertNormalizer( + clean_text=True, + handle_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + lowercase=do_lower_case, + ) + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + + cls = str(self.original_tokenizer.cls_token) + sep = str(self.original_tokenizer.sep_token) + question = str(self.original_tokenizer.question_token) + dot = "." + cls_token_id = self.original_tokenizer.cls_token_id + sep_token_id = self.original_tokenizer.sep_token_id + question_token_id = self.original_tokenizer.question_token_id + dot_token_id = self.original_tokenizer.convert_tokens_to_ids(".") + + if self.original_tokenizer.padding_side == "right": + pair = f"{cls}:0 $A:0 {question} {dot} {sep}:0 $B:1 {sep}:1" + else: + pair = f"{cls}:0 $A:0 {sep}:0 $B:1 {question} {dot} {sep}:1" + + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{cls}:0 $A:0 {sep}:0", + pair=pair, + special_tokens=[ + (cls, cls_token_id), + (sep, sep_token_id), + (question, question_token_id), + (dot, dot_token_id), + ], + ) + tokenizer.decoder = decoders.WordPiece(prefix="##") + + return tokenizer + + +class FunnelConverter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.vocab + tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token))) + + tokenize_chinese_chars = False + strip_accents = False + do_lower_case = False + if hasattr(self.original_tokenizer, "basic_tokenizer"): + tokenize_chinese_chars = self.original_tokenizer.basic_tokenizer.tokenize_chinese_chars + strip_accents = self.original_tokenizer.basic_tokenizer.strip_accents + do_lower_case = self.original_tokenizer.basic_tokenizer.do_lower_case + + tokenizer.normalizer = normalizers.BertNormalizer( + clean_text=True, + handle_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + lowercase=do_lower_case, + ) + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + + cls = str(self.original_tokenizer.cls_token) + sep = str(self.original_tokenizer.sep_token) + cls_token_id = self.original_tokenizer.cls_token_id + sep_token_id = self.original_tokenizer.sep_token_id + + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{cls}:2 $A:0 {sep}:0", # token_type_id is 2 for Funnel transformer + pair=f"{cls}:2 $A:0 {sep}:0 $B:1 {sep}:1", + special_tokens=[ + (cls, cls_token_id), + (sep, sep_token_id), + ], + ) + tokenizer.decoder = decoders.WordPiece(prefix="##") + + return tokenizer + + +class MPNetConverter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.vocab + tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token))) + + tokenize_chinese_chars = False + strip_accents = False + do_lower_case = False + if hasattr(self.original_tokenizer, "basic_tokenizer"): + tokenize_chinese_chars = self.original_tokenizer.basic_tokenizer.tokenize_chinese_chars + strip_accents = self.original_tokenizer.basic_tokenizer.strip_accents + do_lower_case = self.original_tokenizer.basic_tokenizer.do_lower_case + + tokenizer.normalizer = normalizers.BertNormalizer( + clean_text=True, + handle_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + lowercase=do_lower_case, + ) + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + + cls = str(self.original_tokenizer.cls_token) + sep = str(self.original_tokenizer.sep_token) + cls_token_id = self.original_tokenizer.cls_token_id + sep_token_id = self.original_tokenizer.sep_token_id + + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{cls}:0 $A:0 {sep}:0", + pair=f"{cls}:0 $A:0 {sep}:0 {sep}:0 $B:1 {sep}:1", # MPNet uses two [SEP] tokens + special_tokens=[ + (cls, cls_token_id), + (sep, sep_token_id), + ], + ) + tokenizer.decoder = decoders.WordPiece(prefix="##") + + return tokenizer + + +class OpenAIGPTConverter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.encoder + merges = list(self.original_tokenizer.bpe_ranks.keys()) + unk_token = self.original_tokenizer.unk_token + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + unk_token=str(unk_token), + end_of_word_suffix="", + fuse_unk=False, + ) + ) + + if tokenizer.token_to_id(str(unk_token)) is not None: + tokenizer.add_special_tokens([str(unk_token)]) + + tokenizer.normalizer = normalizers.BertNormalizer(lowercase=True) + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + tokenizer.decoder = decoders.BPEDecoder(suffix="") + + return tokenizer + + +class GPT2Converter(Converter): + def converted(self, vocab: dict[str, int] = None, merges: list[tuple[str, str]] = None) -> Tokenizer: + if not vocab: + vocab = self.original_tokenizer.encoder + if not merges: + merges = list(self.original_tokenizer.bpe_ranks) + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + ) + ) + + add_prefix_space = getattr(self.original_tokenizer, "add_prefix_space", False) + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=add_prefix_space) + tokenizer.decoder = decoders.ByteLevel() + if getattr(self.original_tokenizer, "add_bos_token", False): + bos = self.original_tokenizer.bos_token + bos_token_id = self.original_tokenizer.bos_token_id + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{bos}:0 $A:0", + pair=f"{bos}:0 $A:0 $B:1", + special_tokens=[ + (bos, bos_token_id), + ], + ) + else: + # XXX trim_offsets=False actually means this post_processor doesn't + # really do anything. + tokenizer.post_processor = processors.ByteLevel(trim_offsets=False) + return tokenizer + + +class HerbertConverter(Converter): + def converted(self) -> Tokenizer: + tokenizer_info_str = "#version:" + token_suffix = "" + + vocab = self.original_tokenizer.encoder + merges = list(self.original_tokenizer.bpe_ranks.keys()) + if tokenizer_info_str in merges[0][0]: + merges = merges[1:] + + tokenizer = Tokenizer( + BPE( + vocab, + merges, + dropout=None, + unk_token=self.original_tokenizer.unk_token, + end_of_word_suffix=token_suffix, + ) + ) + + tokenizer.normalizer = normalizers.BertNormalizer(lowercase=False, strip_accents=False) + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + tokenizer.decoder = decoders.BPEDecoder(suffix=token_suffix) + tokenizer.post_processor = processors.BertProcessing( + sep=(self.original_tokenizer.sep_token, self.original_tokenizer.sep_token_id), + cls=(self.original_tokenizer.cls_token, self.original_tokenizer.cls_token_id), + ) + + return tokenizer + + +class Qwen2Converter(Converter): + def converted(self, vocab: dict[str, int] = None, merges: list[tuple[str, str]] = None) -> Tokenizer: + if not vocab: + vocab = self.original_tokenizer.encoder + if not merges: + merges = list(self.original_tokenizer.bpe_ranks.keys()) + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + unk_token=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + byte_fallback=False, + ) + ) + + tokenizer.normalizer = normalizers.NFC() + + tokenizer.pre_tokenizer = pre_tokenizers.Sequence( + [ + pre_tokenizers.Split( + Regex( + r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""" + ), + behavior="isolated", + invert=False, + ), + pre_tokenizers.ByteLevel( + add_prefix_space=getattr(self.original_tokenizer, "add_prefix_space", False), + use_regex=False, + ), + ] + ) + + tokenizer.decoder = decoders.ByteLevel() + tokenizer.post_processor = processors.ByteLevel(trim_offsets=False) + + return tokenizer + + +class RobertaConverter(Converter): + def converted(self) -> Tokenizer: + ot = self.original_tokenizer + vocab = ot.encoder + merges = list(ot.bpe_ranks.keys()) + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + ) + ) + + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=ot.add_prefix_space) + tokenizer.decoder = decoders.ByteLevel() + tokenizer.post_processor = processors.RobertaProcessing( + sep=(ot.sep_token, ot.sep_token_id), + cls=(ot.cls_token, ot.cls_token_id), + add_prefix_space=ot.add_prefix_space, + trim_offsets=True, # True by default on Roberta (historical) + ) + + return tokenizer + + +class RoFormerConverter(Converter): + def converted(self) -> Tokenizer: + from .models.roformer.tokenization_utils import JiebaPreTokenizer + + vocab = self.original_tokenizer.vocab + tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token))) + + strip_accents = False + do_lower_case = False + if hasattr(self.original_tokenizer, "basic_tokenizer"): + strip_accents = self.original_tokenizer.basic_tokenizer.strip_accents + do_lower_case = self.original_tokenizer.basic_tokenizer.do_lower_case + + tokenizer.normalizer = normalizers.BertNormalizer( + clean_text=True, + handle_chinese_chars=False, + strip_accents=strip_accents, + lowercase=do_lower_case, + ) + tokenizer.pre_tokenizer = pre_tokenizers.PreTokenizer.custom(JiebaPreTokenizer(vocab)) + + cls = str(self.original_tokenizer.cls_token) + sep = str(self.original_tokenizer.sep_token) + cls_token_id = self.original_tokenizer.cls_token_id + sep_token_id = self.original_tokenizer.sep_token_id + + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{cls}:0 $A:0 {sep}:0", + pair=f"{cls}:0 $A:0 {sep}:0 $B:1 {sep}:1", + special_tokens=[ + (cls, cls_token_id), + (sep, sep_token_id), + ], + ) + tokenizer.decoder = decoders.WordPiece(prefix="##") + + return tokenizer + + +class DebertaConverter(Converter): + def converted(self) -> Tokenizer: + ot = self.original_tokenizer + vocab = ot.encoder + merges = list(ot.bpe_ranks.keys()) + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + ) + ) + + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=ot.add_prefix_space) + tokenizer.decoder = decoders.ByteLevel() + tokenizer.post_processor = processors.TemplateProcessing( + single="[CLS]:0 $A:0 [SEP]:0", + pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1", + special_tokens=[ + ("[CLS]", self.original_tokenizer.convert_tokens_to_ids("[CLS]")), + ("[SEP]", self.original_tokenizer.convert_tokens_to_ids("[SEP]")), + ], + ) + + return tokenizer + + +class SpmConverter(Converter): + handle_byte_fallback = False + SpmExtractor = SentencePieceExtractor + special_tokens = {} + + def __init__(self, *args): + requires_backends(self, "protobuf") + + super().__init__(*args) + + # from .utils import sentencepiece_model_pb2 as model_pb2 + model_pb2 = import_protobuf() + + m = model_pb2.ModelProto() + with open(self.original_tokenizer.vocab_file, "rb") as f: + m.ParseFromString(f.read()) + self.proto = m + + if self.proto.trainer_spec.byte_fallback and not self.handle_byte_fallback: + warnings.warn( + "The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option" + " which is not implemented in the fast tokenizers. In practice this means that the fast version of the" + " tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these " + "unknown tokens into a sequence of byte tokens matching the original piece of text." + ) + + def vocab(self, proto): + return [(piece.piece, piece.score) for piece in proto.pieces] + + def unk_id(self, proto): + return proto.trainer_spec.unk_id + + def tokenizer(self, proto): + model_type = proto.trainer_spec.model_type + vocab_scores = self.vocab(proto) + + if model_type == 1: + tokenizer = Tokenizer( + Unigram( + vocab_scores, + unk_id=self.unk_id(proto), + byte_fallback=self.handle_byte_fallback, + ) + ) + + elif model_type == 2: + _, merges = self.SpmExtractor(self.original_tokenizer.vocab_file).extract(vocab_scores) + bpe_vocab = {word: i for i, (word, score) in enumerate(vocab_scores)} + tokenizer = Tokenizer( + BPE( + bpe_vocab, + merges, + unk_token=proto.trainer_spec.unk_piece, + fuse_unk=True, + byte_fallback=self.handle_byte_fallback, + dropout=None, + ) + ) + + else: + raise Exception( + "You're trying to run a `Unigram` model but you're file was trained with a different algorithm" + ) + + # control tokens are special + # user defined symbols are not + # both user and control tokens are AddedTokens + # Add user defined symbols (type == 4) from sentencepiece (https://github.com/google/sentencepiece/blob/6225e08edb2577757163b3f5dbba4c0b670ef445/src/sentencepiece_model.proto#L299C29-L299C33) + spm_added_tokens = [ + (id, p.piece, p.type == 3 or p.piece in self.special_tokens) + for id, p in enumerate(proto.pieces) + if p.type in [3, 4] + ] + tokenizer.add_tokens( + [ + AddedToken(token, normalized=False, special=special) + for id, token, special in sorted(spm_added_tokens, key=lambda x: x[0]) + ] + ) + + return tokenizer + + def normalizer(self, proto): + precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap + _normalizers = [ + normalizers.Strip(left=False, right=True), # stripping is important + normalizers.Replace(Regex(" {2,}"), "▁"), + ] + if not precompiled_charsmap: + return normalizers.Sequence(_normalizers) + else: + return normalizers.Sequence([normalizers.Precompiled(precompiled_charsmap)] + _normalizers) + + def pre_tokenizer(self, replacement, add_prefix_space): + prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer) + return pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + + def post_processor(self): + return None + + def decoder(self, replacement, add_prefix_space): + prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer) + return decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + + def converted(self) -> Tokenizer: + tokenizer = self.tokenizer(self.proto) + + # Tokenizer assemble + normalizer = self.normalizer(self.proto) + if normalizer is not None: + tokenizer.normalizer = normalizer + + replacement = "▁" + add_prefix_space = True + if hasattr(self.original_tokenizer, "add_prefix_space"): + add_prefix_space = self.original_tokenizer.add_prefix_space + + pre_tokenizer = self.pre_tokenizer(replacement, add_prefix_space) + if pre_tokenizer is not None: + tokenizer.pre_tokenizer = pre_tokenizer + + tokenizer.decoder = self.decoder(replacement, add_prefix_space) + post_processor = self.post_processor() + if post_processor: + tokenizer.post_processor = post_processor + + return tokenizer + + +class AlbertConverter(SpmConverter): + def vocab(self, proto): + return [ + (piece.piece, piece.score) if check_number_comma(piece.piece) else (piece.piece, piece.score - 100) + for piece in proto.pieces + ] + + def normalizer(self, proto): + list_normalizers = [ + normalizers.Replace("``", '"'), + normalizers.Replace("''", '"'), + ] + if not self.original_tokenizer.keep_accents: + list_normalizers.append(normalizers.NFKD()) + list_normalizers.append(normalizers.StripAccents()) + if self.original_tokenizer.do_lower_case: + list_normalizers.append(normalizers.Lowercase()) + + precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap + + if precompiled_charsmap: + list_normalizers.append(normalizers.Precompiled(precompiled_charsmap)) + + list_normalizers.append(normalizers.Replace(Regex(" {2,}"), " ")) + return normalizers.Sequence(list_normalizers) + + def post_processor(self): + return processors.TemplateProcessing( + single="[CLS]:0 $A:0 [SEP]:0", + pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1", + special_tokens=[ + ("[CLS]", self.original_tokenizer.convert_tokens_to_ids("[CLS]")), + ("[SEP]", self.original_tokenizer.convert_tokens_to_ids("[SEP]")), + ], + ) + + +class BarthezConverter(SpmConverter): + def unk_id(self, proto): + unk_id = 3 + return unk_id + + def post_processor(self): + return processors.TemplateProcessing( + single=" $A ", + pair=" $A $B ", + special_tokens=[ + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class CamembertConverter(SpmConverter): + def vocab(self, proto): + vocab = [ + ("NOTUSED", 0.0), + ("", 0.0), + ("NOTUSED", 0.0), + ("", 0.0), + ("NOTUSED", -100), + ] + # We down-grade the original SentencePiece by -100 to avoid using it and use our added token instead + vocab += [(piece.piece, piece.score) for piece in proto.pieces[1:]] + vocab += [("", 0.0)] + return vocab + + def unk_id(self, proto): + # See vocab unk position + return 3 + + def post_processor(self): + return processors.TemplateProcessing( + single=" $A ", + pair=" $A $B ", + special_tokens=[ + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class DebertaV2Converter(SpmConverter): + def pre_tokenizer(self, replacement, add_prefix_space): + list_pretokenizers = [] + if self.original_tokenizer.split_by_punct: + list_pretokenizers.append(pre_tokenizers.Punctuation(behavior="isolated")) + prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer) + list_pretokenizers.append(pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)) + return pre_tokenizers.Sequence(list_pretokenizers) + + def normalizer(self, proto): + list_normalizers = [] + if self.original_tokenizer.do_lower_case: + list_normalizers.append(normalizers.Lowercase()) + list_normalizers.append(normalizers.Strip()) + + precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap + if precompiled_charsmap: + list_normalizers.append(normalizers.Precompiled(precompiled_charsmap)) + list_normalizers.append(normalizers.Replace(Regex(" {2,}"), " ")) + + return normalizers.Sequence(list_normalizers) + + def post_processor(self): + return processors.TemplateProcessing( + single="[CLS]:0 $A:0 [SEP]:0", + pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1", + special_tokens=[ + ("[CLS]", self.original_tokenizer.convert_tokens_to_ids("[CLS]")), + ("[SEP]", self.original_tokenizer.convert_tokens_to_ids("[SEP]")), + ], + ) + + +class MBartConverter(SpmConverter): + def vocab(self, proto): + vocab = [ + ("", 0.0), + ("", 0.0), + ("", 0.0), + ("", 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + vocab += [ + ("ar_AR", 0.0), + ("cs_CZ", 0.0), + ("de_DE", 0.0), + ("en_XX", 0.0), + ("es_XX", 0.0), + ("et_EE", 0.0), + ("fi_FI", 0.0), + ("fr_XX", 0.0), + ("gu_IN", 0.0), + ("hi_IN", 0.0), + ("it_IT", 0.0), + ("ja_XX", 0.0), + ("kk_KZ", 0.0), + ("ko_KR", 0.0), + ("lt_LT", 0.0), + ("lv_LV", 0.0), + ("my_MM", 0.0), + ("ne_NP", 0.0), + ("nl_XX", 0.0), + ("ro_RO", 0.0), + ("ru_RU", 0.0), + ("si_LK", 0.0), + ("tr_TR", 0.0), + ("vi_VN", 0.0), + ("zh_CN", 0.0), + ] + vocab += [("", 0.0)] + return vocab + + def unk_id(self, proto): + return 3 + + def post_processor(self): + return processors.TemplateProcessing( + single="$A en_XX", + pair="$A $B en_XX", + special_tokens=[ + ("en_XX", self.original_tokenizer.convert_tokens_to_ids("en_XX")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class MBart50Converter(SpmConverter): + def vocab(self, proto): + vocab = [ + ("", 0.0), + ("", 0.0), + ("", 0.0), + ("", 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + vocab += [("ar_AR", 0.0), ("cs_CZ", 0.0), ("de_DE", 0.0), ("en_XX", 0.0), ("es_XX", 0.0), ("et_EE", 0.0), ("fi_FI", 0.0), ("fr_XX", 0.0), ("gu_IN", 0.0), ("hi_IN", 0.0), ("it_IT", 0.0), ("ja_XX", 0.0), ("kk_KZ", 0.0), ("ko_KR", 0.0), ("lt_LT", 0.0), ("lv_LV", 0.0), ("my_MM", 0.0), ("ne_NP", 0.0), ("nl_XX", 0.0), ("ro_RO", 0.0), ("ru_RU", 0.0), ("si_LK", 0.0), ("tr_TR", 0.0), ("vi_VN", 0.0), ("zh_CN", 0.0), ("af_ZA", 0.0), ("az_AZ", 0.0), ("bn_IN", 0.0), ("fa_IR", 0.0), ("he_IL", 0.0), ("hr_HR", 0.0), ("id_ID", 0.0), ("ka_GE", 0.0), ("km_KH", 0.0), ("mk_MK", 0.0), ("ml_IN", 0.0), ("mn_MN", 0.0), ("mr_IN", 0.0), ("pl_PL", 0.0), ("ps_AF", 0.0), ("pt_XX", 0.0), ("sv_SE", 0.0), ("sw_KE", 0.0), ("ta_IN", 0.0), ("te_IN", 0.0), ("th_TH", 0.0), ("tl_XX", 0.0), ("uk_UA", 0.0), ("ur_PK", 0.0), ("xh_ZA", 0.0), ("gl_ES", 0.0), ("sl_SI", 0.0)] # fmt: skip + vocab += [("", 0.0)] + return vocab + + def unk_id(self, proto): + return 3 + + def post_processor(self): + return processors.TemplateProcessing( + single="en_XX $A ", + pair="en_XX $A $B ", + special_tokens=[ + ("en_XX", self.original_tokenizer.convert_tokens_to_ids("en_XX")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class NllbConverter(SpmConverter): + def vocab(self, proto): + vocab = [ + ("", 0.0), + ("", 0.0), + ("", 0.0), + ("", 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + return vocab + + def unk_id(self, proto): + return 3 + + def post_processor(self): + return processors.TemplateProcessing( + single="eng_Latn $A ", + pair="eng_Latn $A $B ", + special_tokens=[ + ("eng_Latn", self.original_tokenizer.convert_tokens_to_ids("eng_Latn")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class SeamlessM4TConverter(SpmConverter): + def vocab(self, proto): + vocab = [ + ("", 0.0), + ("", 0.0), + ("", 0.0), + ("", 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + return vocab + + def unk_id(self, proto): + return self.original_tokenizer.unk_token_id + + def post_processor(self): + return processors.TemplateProcessing( + single="__eng__ $A ", + pair="__eng__ $A $B ", + special_tokens=[ + ("__eng__", self.original_tokenizer.convert_tokens_to_ids("__eng__")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class XLMRobertaConverter(SpmConverter): + def vocab(self, proto): + vocab = [ + ("", 0.0), + ("", 0.0), + ("", 0.0), + ("", 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + vocab += [("", 0.0)] + return vocab + + def unk_id(self, proto): + unk_id = 3 + return unk_id + + def post_processor(self): + return processors.TemplateProcessing( + single=" $A ", + pair=" $A $B ", + special_tokens=[ + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class XLNetConverter(SpmConverter): + def vocab(self, proto): + return [ + (piece.piece, piece.score) if check_number_comma(piece.piece) else (piece.piece, piece.score - 100) + for piece in proto.pieces + ] + + def normalizer(self, proto): + list_normalizers = [ + normalizers.Replace("``", '"'), + normalizers.Replace("''", '"'), + ] + if not self.original_tokenizer.keep_accents: + list_normalizers.append(normalizers.NFKD()) + list_normalizers.append(normalizers.StripAccents()) + if self.original_tokenizer.do_lower_case: + list_normalizers.append(normalizers.Lowercase()) + + precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap + + if precompiled_charsmap: + list_normalizers.append(normalizers.Precompiled(precompiled_charsmap)) + + list_normalizers.append(normalizers.Replace(Regex(" {2,}"), " ")) + return normalizers.Sequence(list_normalizers) + + def post_processor(self): + return processors.TemplateProcessing( + single="$A:0 :0 :2", + pair="$A:0 :0 $B:1 :1 :2", + special_tokens=[ + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class ReformerConverter(SpmConverter): + pass + + +class RemBertConverter(SpmConverter): + # Inspired from AlbertConverter + def normalizer(self, proto): + list_normalizers = [ + normalizers.Replace("``", '"'), + normalizers.Replace("''", '"'), + normalizers.Replace(Regex(" {2,}"), " "), + ] + if not self.original_tokenizer.keep_accents: + list_normalizers.append(normalizers.NFKD()) + list_normalizers.append(normalizers.StripAccents()) + if self.original_tokenizer.do_lower_case: + list_normalizers.append(normalizers.Lowercase()) + + precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap + + if precompiled_charsmap: + list_normalizers.append(normalizers.Precompiled(precompiled_charsmap)) + + return normalizers.Sequence(list_normalizers) + + def post_processor(self): + return processors.TemplateProcessing( + single="[CLS]:0 $A:0 [SEP]:0", + pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1", + special_tokens=[ + ("[CLS]", self.original_tokenizer.convert_tokens_to_ids("[CLS]")), + ("[SEP]", self.original_tokenizer.convert_tokens_to_ids("[SEP]")), + ], + ) + + +class BertGenerationConverter(SpmConverter): + pass + + +class PegasusConverter(SpmConverter): + def vocab(self, proto): + vocab = [ + (self.original_tokenizer.pad_token, 0.0), + (self.original_tokenizer.eos_token, 0.0), + ] + + if self.original_tokenizer.mask_token_sent is not None: + vocab += [(self.original_tokenizer.mask_token_sent, 0.0)] + + if ( + self.original_tokenizer.mask_token is not None + and self.original_tokenizer.mask_token_id < self.original_tokenizer.offset + ): + vocab += [(self.original_tokenizer.mask_token, 0.0)] + + vocab += [(f"", -100.0) for i in range(2, self.original_tokenizer.offset)] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[2:]] + return vocab + + def unk_id(self, proto): + return proto.trainer_spec.unk_id + self.original_tokenizer.offset + + def pre_tokenizer(self, replacement, add_prefix_space): + prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer) + return pre_tokenizers.Sequence( + [ + pre_tokenizers.WhitespaceSplit(), + pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme), + ] + ) + + def post_processor(self): + eos = self.original_tokenizer.eos_token + special_tokens = [ + (eos, self.original_tokenizer.eos_token_id), + ] + return processors.TemplateProcessing(single=["$A", eos], pair=["$A", "$B", eos], special_tokens=special_tokens) + + +class T5Converter(SpmConverter): + def vocab(self, proto): + num_extra_ids = self.original_tokenizer._extra_ids + vocab = [(piece.piece, piece.score) for piece in proto.pieces] + vocab += [(f"", 0.0) for i in range(num_extra_ids - 1, -1, -1)] + return vocab + + def post_processor(self): + return processors.TemplateProcessing( + single=["$A", ""], + pair=["$A", "", "$B", ""], + special_tokens=[ + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class UdopConverter(SpmConverter): + def post_processor(self): + return processors.TemplateProcessing( + single=["$A", ""], + pair=["$A", "", "$B", ""], + special_tokens=[ + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class WhisperConverter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.encoder + merges = list(self.original_tokenizer.bpe_ranks.keys()) + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + ) + ) + + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=self.original_tokenizer.add_prefix_space) + tokenizer.decoder = decoders.ByteLevel() + + prefix_token_ids = self.original_tokenizer.prefix_tokens + prefixes = self.original_tokenizer.convert_ids_to_tokens(prefix_token_ids) + eos = self.original_tokenizer.eos_token + eos_token_id = self.original_tokenizer.eos_token_id + prefix_template = " ".join([f"{token}:0" for token in prefixes]) + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{prefix_template} $A:0 {eos}:0", + pair=f"{prefix_template} $A:0 $B:1 {eos}:1", + special_tokens=[ + (eos, eos_token_id), + *zip(prefixes, prefix_token_ids), + ], + ) + + return tokenizer + + +class BigBirdConverter(SpmConverter): + def post_processor(self): + return processors.TemplateProcessing( + single="[CLS]:0 $A:0 [SEP]:0", + pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1", + special_tokens=[ + ("[CLS]", self.original_tokenizer.convert_tokens_to_ids("[CLS]")), + ("[SEP]", self.original_tokenizer.convert_tokens_to_ids("[SEP]")), + ], + ) + + +class CLIPConverter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.encoder + merges = list(self.original_tokenizer.bpe_ranks.keys()) + unk_token = self.original_tokenizer.unk_token + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + unk_token=str(unk_token), + ) + ) + + tokenizer.normalizer = normalizers.Sequence( + [normalizers.NFC(), normalizers.Replace(Regex(r"\s+"), " "), normalizers.Lowercase()] + ) + tokenizer.pre_tokenizer = pre_tokenizers.Sequence( + [ + pre_tokenizers.Split( + Regex(r"""'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+"""), + behavior="removed", + invert=True, + ), + pre_tokenizers.ByteLevel(add_prefix_space=False), + ] + ) + tokenizer.decoder = decoders.ByteLevel() + + # Hack to have a ByteLevel and TemplaceProcessor + tokenizer.post_processor = processors.RobertaProcessing( + sep=(self.original_tokenizer.eos_token, self.original_tokenizer.eos_token_id), + cls=(self.original_tokenizer.bos_token, self.original_tokenizer.bos_token_id), + add_prefix_space=False, + trim_offsets=False, + ) + return tokenizer + + +class LayoutLMv2Converter(Converter): + def converted(self) -> Tokenizer: + vocab = self.original_tokenizer.vocab + tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token))) + + tokenize_chinese_chars = False + strip_accents = False + do_lower_case = True + if hasattr(self.original_tokenizer, "basic_tokenizer"): + tokenize_chinese_chars = self.original_tokenizer.basic_tokenizer.tokenize_chinese_chars + strip_accents = self.original_tokenizer.basic_tokenizer.strip_accents + do_lower_case = self.original_tokenizer.basic_tokenizer.do_lower_case + + tokenizer.normalizer = normalizers.BertNormalizer( + clean_text=True, + handle_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + lowercase=do_lower_case, + ) + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + + cls = str(self.original_tokenizer.cls_token) + sep = str(self.original_tokenizer.sep_token) + cls_token_id = self.original_tokenizer.cls_token_id + sep_token_id = self.original_tokenizer.sep_token_id + + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{cls}:0 $A:0 {sep}:0", + pair=f"{cls}:0 $A:0 {sep}:0 $B:1 {sep}:1", + special_tokens=[ + (cls, cls_token_id), + (sep, sep_token_id), + ], + ) + tokenizer.decoder = decoders.WordPiece(prefix="##") + + return tokenizer + + +class BlenderbotConverter(Converter): + def converted(self) -> Tokenizer: + ot = self.original_tokenizer + vocab = ot.encoder + merges = list(ot.bpe_ranks.keys()) + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + ) + ) + + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=ot.add_prefix_space) + tokenizer.decoder = decoders.ByteLevel() + tokenizer.post_processor = processors.TemplateProcessing( + single=f"$A:0 {ot.eos_token}:0", + special_tokens=[ + (ot.eos_token, ot.eos_token_id), + ], + ) + + return tokenizer + + +class XGLMConverter(SpmConverter): + def vocab(self, proto): + vocab = [ + ("", 0.0), + ("", 0.0), + ("", 0.0), + ("", 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + vocab += [("", 0.0), ("", 0.0), ("", 0.0), ("", 0.0), ("", 0.0), ("", 0.0), ("", 0.0)] # fmt: skip + return vocab + + def unk_id(self, proto): + unk_id = 3 + return unk_id + + def post_processor(self): + return processors.TemplateProcessing( + single=" $A", + pair=" $A $B", + special_tokens=[ + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ("", self.original_tokenizer.convert_tokens_to_ids("")), + ], + ) + + +class GemmaConverter(SpmConverter): + handle_byte_fallback = True + SpmExtractor = GemmaSentencePieceExtractor + # start and end of turn tokens must be marked as special + special_tokens = {"", ""} + + """" + split_by_unicode_script: true + split_by_number: true + split_by_whitespace: true + treat_whitespace_as_suffix: false + allow_whitespace_only_pieces: true + split_digits: true + byte_fallback: true + """ + + def normalizer(self, proto): + return normalizers.Replace(" ", "▁") + + def vocab(self, proto): + vocab = [ + (self.original_tokenizer.pad_token, 0.0), + (self.original_tokenizer.eos_token, 0.0), + (self.original_tokenizer.bos_token, 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + + # Older gemma tokenizers had a missing tab token, so we fix that here + if not any(x[0] == "\t" for x in vocab): + override_index = next((i for i, x in enumerate(vocab) if x[0] == "<0x09>"), None) + if override_index is not None: + vocab[override_index] = ("\t", 0.0) + + return vocab + + def pre_tokenizer(self, replacement, add_prefix_space): + return pre_tokenizers.Split(" ", "merged_with_previous") + + def unk_id(self, proto): + unk_id = 3 + return unk_id + + def decoder(self, replacement, add_prefix_space): + return decoders.Sequence( + [ + decoders.Replace("▁", " "), + decoders.ByteFallback(), + decoders.Fuse(), + ] + ) + + +class LlamaConverter(SpmConverter): + handle_byte_fallback = True + + def vocab(self, proto): + vocab = [ + (self.original_tokenizer.convert_ids_to_tokens(0), 0.0), + (self.original_tokenizer.convert_ids_to_tokens(1), 0.0), + (self.original_tokenizer.convert_ids_to_tokens(2), 0.0), + ] + vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] + return vocab + + def unk_id(self, proto): + unk_id = 0 + return unk_id + + def decoder(self, replacement, add_prefix_space): + sequence = [ + decoders.Replace("▁", " "), + decoders.ByteFallback(), + decoders.Fuse(), + ] + if add_prefix_space: + sequence += [decoders.Strip(content=" ", left=1)] + return decoders.Sequence(sequence) + + def normalizer(self, proto): + if getattr(self.original_tokenizer, "legacy", True): + sequence = [] + if getattr(self.original_tokenizer, "add_prefix_space", True): + sequence += [normalizers.Prepend(prepend="▁")] + sequence += [normalizers.Replace(pattern=" ", content="▁")] + return normalizers.Sequence(sequence) + return None # non-legacy, no normalizer + + def pre_tokenizer(self, replacement, add_prefix_space): + if not getattr(self.original_tokenizer, "legacy", True): # non-legacy, we need a replace + prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer) + return pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme, split=False) + return None + + def post_processor(self): + # the processor is defined in the LlamaTokenizerFast class. + return None + + +class MarkupLMConverter(Converter): + def converted(self) -> Tokenizer: + ot = self.original_tokenizer + vocab = ot.encoder + merges = list(ot.bpe_ranks.keys()) + + tokenizer = Tokenizer( + BPE( + vocab=vocab, + merges=merges, + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + unk_token=self.original_tokenizer.unk_token, + ) + ) + + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=ot.add_prefix_space) + tokenizer.decoder = decoders.ByteLevel() + + cls = str(self.original_tokenizer.cls_token) + sep = str(self.original_tokenizer.sep_token) + cls_token_id = self.original_tokenizer.cls_token_id + sep_token_id = self.original_tokenizer.sep_token_id + + tokenizer.post_processor = processors.TemplateProcessing( + single=f"{cls} $A {sep}", + pair=f"{cls} $A {sep} $B {sep}", + special_tokens=[ + (cls, cls_token_id), + (sep, sep_token_id), + ], + ) + + return tokenizer + + +class MoshiConverter(SpmConverter): + handle_byte_fallback = True + + def __init__(self, vocab_file, model_max_length=None, **kwargs): + requires_backends(self, "protobuf") + + Converter.__init__(self, vocab_file) + + # from .utils import sentencepiece_model_pb2 as model_pb2 + model_pb2 = import_protobuf() + + m = model_pb2.ModelProto() + with open(vocab_file, "rb") as f: + m.ParseFromString(f.read()) + self.proto = m + + def normalizer(self, proto): + precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap + _normalizers = [ + normalizers.Replace(" ", "▁"), + ] + if not precompiled_charsmap: + return normalizers.Sequence(_normalizers) + else: + return normalizers.Sequence([normalizers.Precompiled(precompiled_charsmap)] + _normalizers) + + def decoder(self, replacement, add_prefix_space): + sequence = [ + decoders.Replace("▁", " "), + decoders.ByteFallback(), + decoders.Fuse(), + ] + if add_prefix_space: + sequence += [decoders.Strip(content=" ", left=1)] + return decoders.Sequence(sequence) + + def pre_tokenizer(self, replacement, add_prefix_space): + prepend_scheme = "first" + return pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme, split=False) + + +class HeliumConverter(SpmConverter): + handle_byte_fallback = True + + def __init__(self, vocab_file=None, *args): + requires_backends(self, "protobuf") + + Converter.__init__(self, vocab_file) + + model_pb2 = import_protobuf() + + m = model_pb2.ModelProto() + with open(vocab_file, "rb") as f: + m.ParseFromString(f.read()) + self.proto = m + + def tokenizer(self, proto): + vocab_scores = self.vocab(proto) + tokenizer = Tokenizer( + Unigram( + vocab_scores, + unk_id=self.unk_id(proto), + byte_fallback=self.handle_byte_fallback, + ) + ) + # control tokens are special + # user defined symbols are not + # both user and control tokens are AddedTokens + # Add user defined symbols (type == 4) from sentencepiece (https://github.com/google/sentencepiece/blob/6225e08edb2577757163b3f5dbba4c0b670ef445/src/sentencepiece_model.proto#L299C29-L299C33) + spm_added_tokens = [ + (id, p.piece, p.type == 3 or p.piece in self.special_tokens) + for id, p in enumerate(proto.pieces) + if p.type in [3, 4] + ] + tokenizer.add_tokens( + [ + AddedToken(token, normalized=False, special=special, single_word=True) + for id, token, special in sorted(spm_added_tokens, key=lambda x: x[0]) + ] + ) + tokenizer.add_tokens([AddedToken("\n", normalized=False, special=False)]) + tokenizer.enable_padding(pad_token="", pad_id=3) + return tokenizer + + def vocab(self, proto): + vocab = [] + for piece in proto.pieces: + if piece.piece == "<0x0A>": + vocab += [("\n", piece.score)] + else: + vocab += [(piece.piece, piece.score)] + return vocab + + def unk_id(self, proto): + unk_id = 0 + return unk_id + + def decoder(self, replacement, add_prefix_space): + sequence = [ + decoders.Replace("▁", " "), + decoders.ByteFallback(), + decoders.Fuse(), + ] + sequence += [decoders.Strip(content=" ", left=1)] + return decoders.Sequence(sequence) + + def normalizer(self, proto): + return normalizers.Sequence([normalizers.Prepend(" "), normalizers.Replace(r" ", "▁")]) + + def pre_tokenizer(self, replacement, add_prefix_space): + return pre_tokenizers.Sequence([pre_tokenizers.Split("\n", "contiguous")]) + + def post_processor(self): + return processors.TemplateProcessing( + single=[ + "", + "$A", + ], + pair=[ + "", + "$A", + "", + "$B", + ], + special_tokens=[ + ("", 1), + ], + ) + + +# Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control + characters the bpe code barfs on. + + The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab + if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for + decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup + tables between utf-8 bytes and unicode strings. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +class TikTokenConverter: + """ + A general tiktoken converter. + """ + + def __init__( + self, + vocab_file=None, + pattern=r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""", + add_prefix_space=False, + additional_special_tokens=None, + *args, + **kwargs, + ): + super().__init__(*args) + self.vocab_file = vocab_file + self.pattern = pattern + self.add_prefix_space = add_prefix_space + self.additional_special_tokens = ( + additional_special_tokens.keys() if type(additional_special_tokens) is dict else additional_special_tokens + ) + + def extract_vocab_merges_from_model(self, tiktoken_url: str): + try: + from tiktoken.load import load_tiktoken_bpe + except Exception: + raise ValueError( + "`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`." + ) + + bpe_ranks = load_tiktoken_bpe(tiktoken_url) + byte_encoder = bytes_to_unicode() + + def token_bytes_to_string(b): + return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")]) + + merges = [] + vocab = {} + for token, rank in bpe_ranks.items(): + vocab[token_bytes_to_string(token)] = rank + if len(token) == 1: + continue + local = [] + for index in range(1, len(token)): + piece_l, piece_r = token[:index], token[index:] + if piece_l in bpe_ranks and piece_r in bpe_ranks and (piece_l + piece_r) in bpe_ranks: + local.append((piece_l, piece_r, rank)) + local = sorted(local, key=lambda x: (bpe_ranks[x[0]], bpe_ranks[x[1]]), reverse=False) + merges.extend(local) + merges = sorted(merges, key=lambda val: val[2], reverse=False) + merges = [(token_bytes_to_string(val[0]), token_bytes_to_string(val[1])) for val in merges] + return vocab, merges + + def tokenizer(self): + vocab_scores, merges = self.extract_vocab_merges_from_model(self.vocab_file) + tokenizer = Tokenizer(BPE(vocab_scores, merges, fuse_unk=False)) + if hasattr(tokenizer.model, "ignore_merges"): + tokenizer.model.ignore_merges = True + return tokenizer + + def converted(self) -> Tokenizer: + tokenizer = self.tokenizer() + tokenizer.pre_tokenizer = pre_tokenizers.Sequence( + [ + pre_tokenizers.Split(Regex(self.pattern), behavior="isolated", invert=False), + pre_tokenizers.ByteLevel(add_prefix_space=self.add_prefix_space, use_regex=False), + ] + ) + tokenizer.decoder = decoders.ByteLevel() + + tokenizer.add_special_tokens( + [AddedToken(token, normalized=False, special=True) for token in self.additional_special_tokens] + ) + + tokenizer.post_processor = processors.ByteLevel(trim_offsets=False) + + return tokenizer + + +SLOW_TO_FAST_CONVERTERS = { + "AlbertTokenizer": AlbertConverter, + "BartTokenizer": RobertaConverter, + "BarthezTokenizer": BarthezConverter, + "BertTokenizer": BertConverter, + "BigBirdTokenizer": BigBirdConverter, + "BlenderbotTokenizer": BlenderbotConverter, + "CamembertTokenizer": CamembertConverter, + "CLIPTokenizer": CLIPConverter, + "CodeGenTokenizer": GPT2Converter, + "ConvBertTokenizer": BertConverter, + "DebertaTokenizer": DebertaConverter, + "DebertaV2Tokenizer": DebertaV2Converter, + "DistilBertTokenizer": BertConverter, + "DPRReaderTokenizer": BertConverter, + "DPRQuestionEncoderTokenizer": BertConverter, + "DPRContextEncoderTokenizer": BertConverter, + "ElectraTokenizer": BertConverter, + "FNetTokenizer": AlbertConverter, + "FunnelTokenizer": FunnelConverter, + "GPT2Tokenizer": GPT2Converter, + "HerbertTokenizer": HerbertConverter, + "LayoutLMTokenizer": BertConverter, + "LayoutLMv2Tokenizer": BertConverter, + "LayoutLMv3Tokenizer": RobertaConverter, + "LayoutXLMTokenizer": XLMRobertaConverter, + "LongformerTokenizer": RobertaConverter, + "LEDTokenizer": RobertaConverter, + "LxmertTokenizer": BertConverter, + "MarkupLMTokenizer": MarkupLMConverter, + "MBartTokenizer": MBartConverter, + "MBart50Tokenizer": MBart50Converter, + "MPNetTokenizer": MPNetConverter, + "MobileBertTokenizer": BertConverter, + "MvpTokenizer": RobertaConverter, + "NllbTokenizer": NllbConverter, + "OpenAIGPTTokenizer": OpenAIGPTConverter, + "PegasusTokenizer": PegasusConverter, + "Qwen2Tokenizer": Qwen2Converter, + "RealmTokenizer": BertConverter, + "ReformerTokenizer": ReformerConverter, + "RemBertTokenizer": RemBertConverter, + "RetriBertTokenizer": BertConverter, + "RobertaTokenizer": RobertaConverter, + "RoFormerTokenizer": RoFormerConverter, + "SeamlessM4TTokenizer": SeamlessM4TConverter, + "SqueezeBertTokenizer": BertConverter, + "T5Tokenizer": T5Converter, + "UdopTokenizer": UdopConverter, + "WhisperTokenizer": WhisperConverter, + "XLMRobertaTokenizer": XLMRobertaConverter, + "XLNetTokenizer": XLNetConverter, + "SplinterTokenizer": SplinterConverter, + "XGLMTokenizer": XGLMConverter, + "LlamaTokenizer": LlamaConverter, + "CodeLlamaTokenizer": LlamaConverter, + "GemmaTokenizer": GemmaConverter, + "Phi3Tokenizer": LlamaConverter, +} + + +def convert_slow_tokenizer(transformer_tokenizer, from_tiktoken=False) -> Tokenizer: + """ + Utilities to convert a slow tokenizer instance in a fast tokenizer instance. + + Args: + transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]): + Instance of a slow tokenizer to convert in the backend tokenizer for + [`~tokenization_utils_base.PreTrainedTokenizerFast`]. + from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece. + Defaults to False. + + Return: + A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a + [`~tokenization_utils_base.PreTrainedTokenizerFast`] + """ + + tokenizer_class_name = transformer_tokenizer.__class__.__name__ + if tokenizer_class_name in SLOW_TO_FAST_CONVERTERS and not from_tiktoken: + converter_class = SLOW_TO_FAST_CONVERTERS[tokenizer_class_name] + return converter_class(transformer_tokenizer).converted() + + else: + try: + logger.info("Converting from Tiktoken") + return TikTokenConverter( + vocab_file=transformer_tokenizer.vocab_file, + additional_special_tokens=transformer_tokenizer.additional_special_tokens, + ).converted() + except Exception: + raise ValueError( + f"Converting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path " + f"with a SentencePiece tokenizer.model file." + f"Currently available slow->fast converters: {list(SLOW_TO_FAST_CONVERTERS.keys())}" + ) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/data_collator.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/data_collator.py new file mode 100644 index 0000000000000000000000000000000000000000..07490a25f9e58682058732f7a7b13601afff7e73 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/data_collator.py @@ -0,0 +1,2011 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing as mp +import random +import warnings +from collections.abc import Mapping +from dataclasses import dataclass +from random import randint +from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union + +import numpy as np + +from ..models.bert import BertTokenizer, BertTokenizerFast +from ..tokenization_utils_base import PreTrainedTokenizerBase +from ..utils import PaddingStrategy + + +InputDataClass = NewType("InputDataClass", Any) + +""" +A DataCollator is a function that takes a list of samples from a Dataset and collate them into a batch, as a dictionary +of PyTorch/TensorFlow tensors or NumPy arrays. +""" +DataCollator = NewType("DataCollator", Callable[[List[InputDataClass]], Dict[str, Any]]) + + +class DataCollatorMixin: + def __call__(self, features, return_tensors=None): + if return_tensors is None: + return_tensors = self.return_tensors + if return_tensors == "tf": + return self.tf_call(features) + elif return_tensors == "pt": + return self.torch_call(features) + elif return_tensors == "np": + return self.numpy_call(features) + else: + raise ValueError(f"Framework '{return_tensors}' not recognized!") + + +def pad_without_fast_tokenizer_warning(tokenizer, *pad_args, **pad_kwargs): + """ + Pads without triggering the warning about how using the pad function is sub-optimal when using a fast tokenizer. + """ + + # To avoid errors when using Feature extractors + if not hasattr(tokenizer, "deprecation_warnings"): + return tokenizer.pad(*pad_args, **pad_kwargs) + + # Save the state of the warning, then disable it + warning_state = tokenizer.deprecation_warnings.get("Asking-to-pad-a-fast-tokenizer", False) + tokenizer.deprecation_warnings["Asking-to-pad-a-fast-tokenizer"] = True + + try: + padded = tokenizer.pad(*pad_args, **pad_kwargs) + finally: + # Restore the state of the warning. + tokenizer.deprecation_warnings["Asking-to-pad-a-fast-tokenizer"] = warning_state + + return padded + + +def default_data_collator(features: List[InputDataClass], return_tensors="pt") -> Dict[str, Any]: + """ + Very simple data collator that simply collates batches of dict-like objects and performs special handling for + potential keys named: + + - `label`: handles a single value (int or float) per object + - `label_ids`: handles a list of values per object + + Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs + to the model. See glue and ner for example of how it's useful. + """ + + # In this function we'll make the assumption that all `features` in the batch + # have the same attributes. + # So we will look at the first element as a proxy for what attributes exist + # on the whole batch. + + if return_tensors == "pt": + return torch_default_data_collator(features) + elif return_tensors == "tf": + return tf_default_data_collator(features) + elif return_tensors == "np": + return numpy_default_data_collator(features) + + +@dataclass +class DefaultDataCollator(DataCollatorMixin): + """ + Very simple data collator that simply collates batches of dict-like objects and performs special handling for + potential keys named: + + - `label`: handles a single value (int or float) per object + - `label_ids`: handles a list of values per object + + Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs + to the model. See glue and ner for example of how it's useful. + + This is an object (like other data collators) rather than a pure function like default_data_collator. This can be + helpful if you need to set a return_tensors value at initialization. + + Args: + return_tensors (`str`, *optional*, defaults to `"pt"`): + The type of Tensor to return. Allowable values are "np", "pt" and "tf". + """ + + return_tensors: str = "pt" + + def __call__(self, features: List[Dict[str, Any]], return_tensors=None) -> Dict[str, Any]: + if return_tensors is None: + return_tensors = self.return_tensors + return default_data_collator(features, return_tensors) + + +def torch_default_data_collator(features: List[InputDataClass]) -> Dict[str, Any]: + import torch + + if not isinstance(features[0], Mapping): + features = [vars(f) for f in features] + first = features[0] + batch = {} + + # Special handling for labels. + # Ensure that tensor is created with the correct type + # (it should be automatically the case, but let's make sure of it.) + if "label" in first and first["label"] is not None: + label = first["label"].item() if isinstance(first["label"], torch.Tensor) else first["label"] + dtype = torch.long if isinstance(label, int) else torch.float + batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype) + elif "label_ids" in first and first["label_ids"] is not None: + if isinstance(first["label_ids"], torch.Tensor): + batch["labels"] = torch.stack([f["label_ids"] for f in features]) + else: + dtype = torch.long if isinstance(first["label_ids"][0], int) else torch.float + batch["labels"] = torch.tensor([f["label_ids"] for f in features], dtype=dtype) + + # Handling of all other possible keys. + # Again, we will use the first element to figure out which key/values are not None for this model. + for k, v in first.items(): + if k not in ("label", "label_ids") and v is not None and not isinstance(v, str): + if isinstance(v, torch.Tensor): + batch[k] = torch.stack([f[k] for f in features]) + elif isinstance(v, np.ndarray): + batch[k] = torch.from_numpy(np.stack([f[k] for f in features])) + else: + batch[k] = torch.tensor([f[k] for f in features]) + + return batch + + +def tf_default_data_collator(features: List[InputDataClass]) -> Dict[str, Any]: + import tensorflow as tf + + if not isinstance(features[0], Mapping): + features = [vars(f) for f in features] + first = features[0] + batch = {} + + # Special handling for labels. + # Ensure that tensor is created with the correct type + # (it should be automatically the case, but let's make sure of it.) + if "label" in first and first["label"] is not None: + label_col_name = "label" + elif "label_ids" in first and first["label_ids"] is not None: + label_col_name = "label_ids" + elif "labels" in first and first["labels"] is not None: + label_col_name = "labels" + else: + label_col_name = None + if label_col_name is not None: + if isinstance(first[label_col_name], tf.Tensor): + dtype = tf.int64 if first[label_col_name].dtype.is_integer else tf.float32 + elif isinstance(first[label_col_name], np.ndarray) or isinstance(first[label_col_name], np.generic): + dtype = tf.int64 if np.issubdtype(first[label_col_name].dtype, np.integer) else tf.float32 + elif isinstance(first[label_col_name], (tuple, list)): + dtype = tf.int64 if isinstance(first[label_col_name][0], int) else tf.float32 + else: + dtype = tf.int64 if isinstance(first[label_col_name], int) else tf.float32 + batch["labels"] = tf.convert_to_tensor([f[label_col_name] for f in features], dtype=dtype) + # Handling of all other possible keys. + # Again, we will use the first element to figure out which key/values are not None for this model. + for k, v in first.items(): + if k not in ("label", "label_ids", "labels") and v is not None and not isinstance(v, str): + if isinstance(v, (tf.Tensor, np.ndarray)): + batch[k] = tf.stack([f[k] for f in features]) + else: + batch[k] = tf.convert_to_tensor([f[k] for f in features]) + + return batch + + +def numpy_default_data_collator(features: List[InputDataClass]) -> Dict[str, Any]: + if not isinstance(features[0], Mapping): + features = [vars(f) for f in features] + first = features[0] + batch = {} + + # Special handling for labels. + # Ensure that tensor is created with the correct type + # (it should be automatically the case, but let's make sure of it.) + if "label" in first and first["label"] is not None: + label = first["label"].item() if isinstance(first["label"], np.ndarray) else first["label"] + dtype = np.int64 if isinstance(label, int) else np.float32 + batch["labels"] = np.array([f["label"] for f in features], dtype=dtype) + elif "label_ids" in first and first["label_ids"] is not None: + if isinstance(first["label_ids"], np.ndarray): + batch["labels"] = np.stack([f["label_ids"] for f in features]) + else: + dtype = np.int64 if isinstance(first["label_ids"][0], int) else np.float32 + batch["labels"] = np.array([f["label_ids"] for f in features], dtype=dtype) + + # Handling of all other possible keys. + # Again, we will use the first element to figure out which key/values are not None for this model. + for k, v in first.items(): + if k not in ("label", "label_ids") and v is not None and not isinstance(v, str): + if isinstance(v, np.ndarray): + batch[k] = np.stack([f[k] for f in features]) + else: + batch[k] = np.array([f[k] for f in features]) + + return batch + + +@dataclass +class DataCollatorWithPadding: + """ + Data collator that will dynamically pad the inputs received. + + Args: + tokenizer ([`PreTrainedTokenizer`] or [`PreTrainedTokenizerFast`]): + The tokenizer used for encoding the data. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding index) + among: + + - `True` or `'longest'` (default): Pad to the longest sequence in the batch (or no padding if only a single + sequence is provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'`: No padding (i.e., can output a batch with sequences of different lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + pad_to_multiple_of (`int`, *optional*): + If set will pad the sequence to a multiple of the provided value. + + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= + 7.0 (Volta). + return_tensors (`str`, *optional*, defaults to `"pt"`): + The type of Tensor to return. Allowable values are "np", "pt" and "tf". + """ + + tokenizer: PreTrainedTokenizerBase + padding: Union[bool, str, PaddingStrategy] = True + max_length: Optional[int] = None + pad_to_multiple_of: Optional[int] = None + return_tensors: str = "pt" + + def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]: + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, + features, + padding=self.padding, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + return_tensors=self.return_tensors, + ) + if "label" in batch: + batch["labels"] = batch["label"] + del batch["label"] + if "label_ids" in batch: + batch["labels"] = batch["label_ids"] + del batch["label_ids"] + return batch + + +@dataclass +class DataCollatorForTokenClassification(DataCollatorMixin): + """ + Data collator that will dynamically pad the inputs received, as well as the labels. + + Args: + tokenizer ([`PreTrainedTokenizer`] or [`PreTrainedTokenizerFast`]): + The tokenizer used for encoding the data. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding index) + among: + + - `True` or `'longest'` (default): Pad to the longest sequence in the batch (or no padding if only a single + sequence is provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'`: No padding (i.e., can output a batch with sequences of different lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + pad_to_multiple_of (`int`, *optional*): + If set will pad the sequence to a multiple of the provided value. + + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= + 7.0 (Volta). + label_pad_token_id (`int`, *optional*, defaults to -100): + The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions). + return_tensors (`str`, *optional*, defaults to `"pt"`): + The type of Tensor to return. Allowable values are "np", "pt" and "tf". + """ + + tokenizer: PreTrainedTokenizerBase + padding: Union[bool, str, PaddingStrategy] = True + max_length: Optional[int] = None + pad_to_multiple_of: Optional[int] = None + label_pad_token_id: int = -100 + return_tensors: str = "pt" + + def torch_call(self, features): + import torch + + label_name = "label" if "label" in features[0].keys() else "labels" + labels = [feature[label_name] for feature in features] if label_name in features[0].keys() else None + + no_labels_features = [{k: v for k, v in feature.items() if k != label_name} for feature in features] + + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, + no_labels_features, + padding=self.padding, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + return_tensors="pt", + ) + + if labels is None: + return batch + + sequence_length = batch["input_ids"].shape[1] + padding_side = self.tokenizer.padding_side + + def to_list(tensor_or_iterable): + if isinstance(tensor_or_iterable, torch.Tensor): + return tensor_or_iterable.tolist() + return list(tensor_or_iterable) + + if padding_side == "right": + batch[label_name] = [ + to_list(label) + [self.label_pad_token_id] * (sequence_length - len(label)) for label in labels + ] + else: + batch[label_name] = [ + [self.label_pad_token_id] * (sequence_length - len(label)) + to_list(label) for label in labels + ] + + batch[label_name] = torch.tensor(batch[label_name], dtype=torch.int64) + return batch + + def tf_call(self, features): + import tensorflow as tf + + label_name = "label" if "label" in features[0].keys() else "labels" + labels = [feature[label_name] for feature in features] if label_name in features[0].keys() else None + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, + features, + padding=self.padding, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + # Conversion to tensors will fail if we have labels as they are not of the same length yet. + return_tensors="tf" if labels is None else None, + ) + + if labels is None: + return batch + + sequence_length = tf.convert_to_tensor(batch["input_ids"]).shape[1] + padding_side = self.tokenizer.padding_side + if padding_side == "right": + batch["labels"] = [ + list(label) + [self.label_pad_token_id] * (sequence_length - len(label)) for label in labels + ] + else: + batch["labels"] = [ + [self.label_pad_token_id] * (sequence_length - len(label)) + list(label) for label in labels + ] + + batch = {k: tf.convert_to_tensor(v, dtype=tf.int64) for k, v in batch.items()} + return batch + + def numpy_call(self, features): + label_name = "label" if "label" in features[0].keys() else "labels" + labels = [feature[label_name] for feature in features] if label_name in features[0].keys() else None + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, + features, + padding=self.padding, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + # Conversion to tensors will fail if we have labels as they are not of the same length yet. + return_tensors="np" if labels is None else None, + ) + + if labels is None: + return batch + + sequence_length = np.array(batch["input_ids"]).shape[1] + padding_side = self.tokenizer.padding_side + if padding_side == "right": + batch["labels"] = [ + list(label) + [self.label_pad_token_id] * (sequence_length - len(label)) for label in labels + ] + else: + batch["labels"] = [ + [self.label_pad_token_id] * (sequence_length - len(label)) + list(label) for label in labels + ] + + batch = {k: np.array(v, dtype=np.int64) for k, v in batch.items()} + return batch + + +def _torch_collate_batch(examples, tokenizer, pad_to_multiple_of: Optional[int] = None): + """Collate `examples` into a batch, using the information in `tokenizer` for padding if necessary.""" + import torch + + # Tensorize if necessary. + if isinstance(examples[0], (list, tuple, np.ndarray)): + examples = [torch.tensor(e, dtype=torch.long) for e in examples] + + length_of_first = examples[0].size(0) + + # Check if padding is necessary. + + are_tensors_same_length = all(x.size(0) == length_of_first for x in examples) + if are_tensors_same_length and (pad_to_multiple_of is None or length_of_first % pad_to_multiple_of == 0): + if not isinstance(examples, torch.Tensor): + return torch.stack(examples, dim=0) + + # If yes, check if we have a `pad_token`. + if tokenizer.pad_token is None: + raise ValueError( + "You are attempting to pad samples but the tokenizer you are using" + f" ({tokenizer.__class__.__name__}) does not have a pad token." + ) + + # Creating the full tensor and filling it with our data. + max_length = max(x.size(0) for x in examples) + if pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0): + max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + result = examples[0].new_full([len(examples), max_length], tokenizer.pad_token_id) + for i, example in enumerate(examples): + if tokenizer.padding_side == "right": + result[i, : example.shape[0]] = example + else: + result[i, -example.shape[0] :] = example + return result + + +def _tf_collate_batch(examples, tokenizer, pad_to_multiple_of: Optional[int] = None): + import tensorflow as tf + + """Collate `examples` into a batch, using the information in `tokenizer` for padding if necessary.""" + # Tensorize if necessary. + if isinstance(examples[0], (list, tuple)): + examples = [tf.convert_to_tensor(e, dtype=tf.int64) for e in examples] + + # Check if padding is necessary. + length_of_first = len(examples[0]) + are_tensors_same_length = all(len(x) == length_of_first for x in examples) + if are_tensors_same_length and (pad_to_multiple_of is None or length_of_first % pad_to_multiple_of == 0): + return tf.stack(examples, axis=0) + + # If yes, check if we have a `pad_token`. + if tokenizer.pad_token is None: + raise ValueError( + "You are attempting to pad samples but the tokenizer you are using" + f" ({tokenizer.__class__.__name__}) does not have a pad token." + ) + + # Creating the full tensor and filling it with our data. + max_length = max(len(x) for x in examples) + if pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0): + max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + # result = examples[0].new_full([len(examples), max_length], tokenizer.pad_token_id) + result = [] + rank = tf.rank(examples[0]) + paddings = np.zeros((rank, 2), dtype=np.int32) + for example in examples: + if tokenizer.padding_side == "right": + paddings[0, 1] = max_length - len(example) + else: + paddings[0, 0] = max_length - len(example) + result.append(tf.pad(example, paddings, constant_values=tokenizer.pad_token_id)) + return tf.stack(result, axis=0) + + +def _numpy_collate_batch(examples, tokenizer, pad_to_multiple_of: Optional[int] = None): + """Collate `examples` into a batch, using the information in `tokenizer` for padding if necessary.""" + # Tensorize if necessary. + if isinstance(examples[0], (list, tuple)): + examples = [np.array(e, dtype=np.int64) for e in examples] + + # Check if padding is necessary. + length_of_first = len(examples[0]) + are_tensors_same_length = all(len(x) == length_of_first for x in examples) + if are_tensors_same_length and (pad_to_multiple_of is None or length_of_first % pad_to_multiple_of == 0): + return np.stack(examples, axis=0) + + # If yes, check if we have a `pad_token`. + if tokenizer.pad_token is None: + raise ValueError( + "You are attempting to pad samples but the tokenizer you are using" + f" ({tokenizer.__class__.__name__}) does not have a pad token." + ) + + # Creating the full tensor and filling it with our data. + max_length = max(len(x) for x in examples) + if pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0): + max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + result = np.full(shape=(len(examples), max_length), fill_value=tokenizer.pad_token_id, dtype=examples[0].dtype) + for i, example in enumerate(examples): + if tokenizer.padding_side == "right": + result[i, : example.shape[0]] = example + else: + result[i, -example.shape[0] :] = example + return result + + +@dataclass +class DataCollatorForMultipleChoice(DataCollatorMixin): + """ + Data collator that dynamically pads a batch of nested examples for multiple choice, so that all choices + of all examples have the same length. + + Args: + tokenizer ([`PreTrainedTokenizer`] or [`PreTrainedTokenizerFast`]): + The tokenizer used for encoding the data. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`): + Select a strategy to pad the returned sequences according to the model's padding side and padding index + among: + + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence + is provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + pad_to_multiple_of (`int`, *optional*): + Pad the sequence to a multiple of the provided value. + + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= + 7.5 (Volta). + return_tensors (`str`, *optional*, defaults to `"pt"`): + The type of Tensor to return. Allowable values are "np", "pt" and "tf". + """ + + tokenizer: PreTrainedTokenizerBase + padding: Union[bool, str, PaddingStrategy] = True + max_length: Optional[int] = None + pad_to_multiple_of: Optional[int] = None + return_tensors: str = "pt" + + def torch_call(self, examples: List[Dict[str, Any]]): # Refactored implementation from the docs. + import torch + + # Take labels out of the examples beforehand, because they aren't nested. + label_name = "label" if "label" in examples[0].keys() else "labels" + labels = [example.pop(label_name) for example in examples] + + batch_size = len(examples) + num_choices = len(examples[0]["input_ids"]) + + # Go from e.g. 2 examples of 2 choices [{input_ids: [[1], [2]]}, {input_ids: [[3], [4]]}] + # to 4 examples [{input_ids: [1]}, {input_ids: [2]}] + [{input_ids: [3]}, {input_ids: [4]}] + flat_examples = sum( + ([{k: v[i] for k, v in example.items()} for i in range(num_choices)] for example in examples), start=[] + ) + + # Pad all choices of all examples as if you're padding any other batch of examples. + batch = self.tokenizer.pad( + flat_examples, + padding=self.padding, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + return_tensors="pt", + ) + + # Reshape from B*C x L into B x C x L, and add the labels back in. + batch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()} + batch["labels"] = torch.tensor(labels, dtype=torch.int64) + return batch + + def tf_call(self, features): # Implementation taken from the docs. + import tensorflow as tf + + label_name = "label" if "label" in features[0].keys() else "labels" + labels = [feature.pop(label_name) for feature in features] + batch_size = len(features) + num_choices = len(features[0]["input_ids"]) + flattened_features = [ + [{k: v[i] for k, v in feature.items()} for i in range(num_choices)] for feature in features + ] + flattened_features = sum(flattened_features, []) # Sometimes written as list(chain(*flattened_features)) + + batch = self.tokenizer.pad( + flattened_features, + padding=self.padding, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + return_tensors="tf", + ) + + batch = {k: tf.reshape(v, (batch_size, num_choices, -1)) for k, v in batch.items()} + batch["labels"] = tf.convert_to_tensor(labels, dtype=tf.int64) + return batch + + +@dataclass +class DataCollatorForSeq2Seq: + """ + Data collator that will dynamically pad the inputs received, as well as the labels. + + Args: + tokenizer ([`PreTrainedTokenizer`] or [`PreTrainedTokenizerFast`]): + The tokenizer used for encoding the data. + model ([`PreTrainedModel`], *optional*): + The model that is being trained. If set and has the *prepare_decoder_input_ids_from_labels*, use it to + prepare the *decoder_input_ids* + + This is useful when using *label_smoothing* to avoid calculating loss twice. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding index) + among: + + - `True` or `'longest'` (default): Pad to the longest sequence in the batch (or no padding if only a single + sequence is provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'`: No padding (i.e., can output a batch with sequences of different lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + pad_to_multiple_of (`int`, *optional*): + If set will pad the sequence to a multiple of the provided value. + + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= + 7.0 (Volta). + label_pad_token_id (`int`, *optional*, defaults to -100): + The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions). + return_tensors (`str`, *optional*, defaults to `"pt"`): + The type of Tensor to return. Allowable values are "np", "pt" and "tf". + """ + + tokenizer: PreTrainedTokenizerBase + model: Optional[Any] = None + padding: Union[bool, str, PaddingStrategy] = True + max_length: Optional[int] = None + pad_to_multiple_of: Optional[int] = None + label_pad_token_id: int = -100 + return_tensors: str = "pt" + + def __call__(self, features, return_tensors=None): + if return_tensors is None: + return_tensors = self.return_tensors + + label_name = "label" if "label" in features[0].keys() else "labels" + labels = [feature[label_name] for feature in features] if label_name in features[0].keys() else None + # reconvert list[None] to None if necessary + # this might occur when we pass {..., "labels": None} + if labels is not None and all(label is None for label in labels): + labels = None + non_labels_features = [{k: v for k, v in feature.items() if k != label_name} for feature in features] + + # run through tokenizer without labels to ensure no side effects + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, + non_labels_features, + padding=self.padding, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + return_tensors=return_tensors, + ) + + # we have to pad the labels manually as we cannot rely on `tokenizer.pad` and we need them to be of the same length to return tensors + no_padding = self.padding is False or self.padding == PaddingStrategy.DO_NOT_PAD + if labels is not None: + if no_padding: + if isinstance(features[0][label_name], list): + batch["labels"] = list(labels) + else: + batch["labels"] = [np.concatenate([label, []]) for label in labels] + else: + max_padding = self.padding == PaddingStrategy.MAX_LENGTH and self.max_length is not None + max_label_length = max(len(l) for l in labels) if not max_padding else self.max_length + if self.pad_to_multiple_of is not None: + max_label_length = ( + (max_label_length + self.pad_to_multiple_of - 1) + // self.pad_to_multiple_of + * self.pad_to_multiple_of + ) + + padding_side = self.tokenizer.padding_side + if isinstance(features[0][label_name], list): + batch["labels"] = [ + label + [self.label_pad_token_id] * (max_label_length - len(label)) + if padding_side == "right" + else [self.label_pad_token_id] * (max_label_length - len(label)) + label + for label in labels + ] + else: + batch["labels"] = [ + np.concatenate( + [ + label, + np.array([self.label_pad_token_id] * (max_label_length - len(label)), dtype=np.int64), + ] + ) + if padding_side == "right" + else np.concatenate( + [ + np.array([self.label_pad_token_id] * (max_label_length - len(label)), dtype=np.int64), + label, + ] + ) + for label in labels + ] + + # reintroduce side effects via tokenizer that return respective datatypes for the `return_tensors` argument + if batch.get("labels", None) is not None: + if return_tensors == "pt": + import torch + + batch["labels"] = torch.tensor(batch["labels"], dtype=torch.int64) + elif return_tensors == "tf": + import tensorflow as tf + + batch["labels"] = tf.constant(batch["labels"], dtype=tf.int64) + else: + batch["labels"] = np.array(batch["labels"], dtype=np.int64) + else: + batch["labels"] = None + + # prepare decoder_input_ids + if ( + labels is not None + and self.model is not None + and hasattr(self.model, "prepare_decoder_input_ids_from_labels") + ): + decoder_input_ids = self.model.prepare_decoder_input_ids_from_labels(labels=batch["labels"]) + batch["decoder_input_ids"] = decoder_input_ids + + return batch + + +@dataclass +class DataCollatorForLanguageModeling(DataCollatorMixin): + """ + Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they + are not all of the same length. + + Args: + tokenizer ([`PreTrainedTokenizer`] or [`PreTrainedTokenizerFast`]): + The tokenizer used for encoding the data. + mlm (`bool`, *optional*, defaults to `True`): + Whether or not to use masked language modeling. If set to `False`, the labels are the same as the inputs + with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked + tokens and the value to predict for the masked token. + mlm_probability (`float`, *optional*, defaults to 0.15): + The probability with which to (randomly) mask tokens in the input, when `mlm` is set to `True`. + mask_replace_prob (`float`, *optional*, defaults to 0.8): + The probability with which masked tokens are replaced by the tokenizer's mask token (e.g., `[MASK]`). + Defaults to 0.8, meaning 80% of the masked tokens will be replaced with `[MASK]`. + Only works when `mlm` is set to `True`. + random_replace_prob (`float`, *optional*, defaults to 0.1): + The probability with which masked tokens are replaced by random tokens from the tokenizer's vocabulary. + Defaults to 0.1, meaning 10% of the masked tokens will be replaced with random tokens. The remaining + masked tokens (1 - mask_replace_prob - random_replace_prob) are left unchanged. + Only works when `mlm` is set to `True`. + pad_to_multiple_of (`int`, *optional*): + If set, will pad the sequence to a multiple of the provided value. + return_tensors (`str`): + The type of Tensor to return. Allowable values are "np", "pt" and "tf". + seed (`int`, *optional*): + The seed to use for the random number generator for masking. If not provided, the global RNG will be used. + + + + For best performance, this data collator should be used with a dataset having items that are dictionaries or + BatchEncoding, with the `"special_tokens_mask"` key, as returned by a [`PreTrainedTokenizer`] or a + [`PreTrainedTokenizerFast`] with the argument `return_special_tokens_mask=True`. + + + + 1. Default Behavior: + - `mask_replace_prob=0.8`, `random_replace_prob=0.1`. + - Expect 80% of masked tokens replaced with `[MASK]`, 10% replaced with random tokens, and 10% left unchanged. + + 2. All masked tokens replaced by `[MASK]`: + - `mask_replace_prob=1.0`, `random_replace_prob=0.0`. + - Expect all masked tokens to be replaced with `[MASK]`. No tokens are left unchanged or replaced with random tokens. + + 3. No `[MASK]` replacement, only random tokens: + - `mask_replace_prob=0.0`, `random_replace_prob=1.0`. + - Expect all masked tokens to be replaced with random tokens. No `[MASK]` replacements or unchanged tokens. + + 4. Balanced replacement: + - `mask_replace_prob=0.5`, `random_replace_prob=0.4`. + - Expect 50% of masked tokens replaced with `[MASK]`, 40% replaced with random tokens, and 10% left unchanged. + + Note: + The sum of `mask_replace_prob` and `random_replace_prob` must not exceed 1. If their sum is less than 1, the + remaining proportion will consist of masked tokens left unchanged. + + + """ + + tokenizer: PreTrainedTokenizerBase + mlm: bool = True + mlm_probability: float = 0.15 + mask_replace_prob: float = 0.8 + random_replace_prob: float = 0.1 + pad_to_multiple_of: Optional[int] = None + tf_experimental_compile: bool = False + return_tensors: str = "pt" + seed: Optional[int] = None + + def __post_init__(self): + if self.mlm and self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for masked language modeling. " + "You should pass `mlm=False` to train on causal language modeling instead." + ) + if self.mlm_probability < 0 or self.mlm_probability > 1: + raise ValueError("mlm_probability should be between 0 and 1.") + if self.mask_replace_prob + self.random_replace_prob > 1: + raise ValueError("The sum of mask_replace_prob and random_replace_prob should not exceed 1") + if self.mask_replace_prob < 0 or self.mask_replace_prob > 1: + raise ValueError("mask_replace_prob should be between 0 and 1.") + if self.random_replace_prob < 0 or self.random_replace_prob > 1: + raise ValueError("random_replace_prob should be between 0 and 1.") + + self.mlm_probability = float(self.mlm_probability) + self.mask_replace_prob = float(self.mask_replace_prob) + self.random_replace_prob = float(self.random_replace_prob) + + if self.tf_experimental_compile: + import tensorflow as tf + + self.tf_mask_tokens = tf.function(self.tf_mask_tokens, jit_compile=True) + + self.generator = None + + def get_generator(self, seed): + if self.return_tensors == "pt": + import torch + + return torch.Generator().manual_seed(seed) + elif self.return_tensors == "tf": + import tensorflow as tf + + return tf.random.Generator.from_seed(seed) + else: + import numpy as np + + return np.random.default_rng(seed) + + def create_rng(self): + if mp.current_process().name == "MainProcess": + # If we are in the main process, we create a generator object with the seed + self.generator = self.get_generator(self.seed) + else: + # If we are in a worker process (i.e using multiprocessing), we need to set a unique seed for each + # worker's generator, generated as the main seed + the worker's ID. + # (https://pytorch.org/docs/stable/data.html#randomness-in-multi-process-data-loading) + # Only PyTorch DataLoader allows us to access the worker ID, and so we check for this. + # For other frameworks, we will throw an error. + import torch + + worker_info = torch.utils.data.get_worker_info() + if worker_info is None: + error_string = ( + "Worker process information is not available for seeding the generator. This may be because", + "you are using multiprocessing without using a PyTorch DataLoader. The `seed` parameter can", + "only be used when using multiprocessing with a PyTorch DataLoader. Please either use a", + "single process or use a PyTorch DataLoader with multiple workers.", + ) + raise ValueError(error_string) + + self.generator = self.get_generator(self.seed + worker_info.id) + + @staticmethod + def tf_bernoulli(shape, probability, generator=None): + import tensorflow as tf + + prob_matrix = tf.fill(shape, probability) + # if generator exists, use it to generate the random numbers + # otherwise, use the global RNG + if generator: + return tf.cast(prob_matrix - generator.uniform(shape, 0, 1) >= 0, tf.bool) + else: + return tf.cast(prob_matrix - tf.random.uniform(shape, 0, 1) >= 0, tf.bool) + + def tf_mask_tokens( + self, inputs: Any, vocab_size, mask_token_id, special_tokens_mask: Optional[Any] = None + ) -> Tuple[Any, Any]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. + """ + import tensorflow as tf + + mask_token_id = tf.cast(mask_token_id, inputs.dtype) + + input_shape = tf.shape(inputs) + # 1 for a special token, 0 for a normal token in the special tokens mask + # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`) + masked_indices = self.tf_bernoulli(input_shape, self.mlm_probability, self.generator) & ~special_tokens_mask + # Replace unmasked indices with -100 in the labels since we only compute loss on masked tokens + labels = tf.where(masked_indices, inputs, -100) + + # mask_replace_prob% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = self.tf_bernoulli(input_shape, self.mask_replace_prob, self.generator) & masked_indices + + inputs = tf.where(indices_replaced, mask_token_id, inputs) + + if self.mask_replace_prob == 1 or self.random_replace_prob == 0: + return inputs, labels + + remaining_prob = 1 - self.mask_replace_prob + # scaling the random_replace_prob to the remaining probability for example if + # mask_replace_prob = 0.8 and random_replace_prob = 0.1, + # then random_replace_prob_scaled = 0.1 / 0.2 = 0.5 + random_replace_prob_scaled = self.random_replace_prob / remaining_prob + # random_replace_prob% of the time, we replace masked input tokens with random word + indices_random = ( + self.tf_bernoulli(input_shape, random_replace_prob_scaled, self.generator) + & masked_indices + & ~indices_replaced + ) + + if self.generator: + random_words = self.generator.uniform(input_shape, maxval=vocab_size, dtype=inputs.dtype) + else: + random_words = tf.random.uniform(input_shape, maxval=vocab_size, dtype=inputs.dtype) + + inputs = tf.where(indices_random, random_words, inputs) + + # The rest of the time ((1-random_replace_prob-mask_replace_prob)% of the time) we keep the masked input tokens unchanged + return inputs, labels + + def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + import tensorflow as tf + + if self.seed and self.generator is None: + # If we have a seed, we need to create a generator object. Subsequent calls to this function will use the same generator. + # If no seed supplied, we will use the global RNG + self.create_rng() + + # Handle dict or lists with proper padding and conversion to tensor. + if isinstance(examples[0], Mapping): + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, examples, return_tensors="tf", pad_to_multiple_of=self.pad_to_multiple_of + ) + else: + batch = { + "input_ids": _tf_collate_batch(examples, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + } + + # If special token mask has been preprocessed, pop it from the dict. + special_tokens_mask = batch.pop("special_tokens_mask", None) + if self.mlm: + if special_tokens_mask is None: + special_tokens_mask = [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) + for val in batch["input_ids"].numpy().tolist() + ] + # Cannot directly create as bool + special_tokens_mask = tf.cast(tf.convert_to_tensor(special_tokens_mask, dtype=tf.int64), tf.bool) + else: + special_tokens_mask = tf.cast(special_tokens_mask, tf.bool) + batch["input_ids"], batch["labels"] = self.tf_mask_tokens( + tf.cast(batch["input_ids"], tf.int64), + special_tokens_mask=special_tokens_mask, + mask_token_id=self.tokenizer.mask_token_id, + vocab_size=len(self.tokenizer), + ) + else: + labels = batch["input_ids"] + if self.tokenizer.pad_token_id is not None: + # Replace self.tokenizer.pad_token_id with -100 + labels = tf.where(labels == self.tokenizer.pad_token_id, -100, labels) + else: + labels = tf.identity(labels) # Makes a copy, just in case + batch["labels"] = labels + return batch + + def torch_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + # Handle dict or lists with proper padding and conversion to tensor. + + if self.seed and self.generator is None: + # If we have a seed, we need to create a generator object. Subsequent calls to this function will use the same generator. + # If no seed supplied, we will use the global RNG + self.create_rng() + + if isinstance(examples[0], Mapping): + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, examples, return_tensors="pt", pad_to_multiple_of=self.pad_to_multiple_of + ) + else: + batch = { + "input_ids": _torch_collate_batch(examples, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + } + + # If special token mask has been preprocessed, pop it from the dict. + special_tokens_mask = batch.pop("special_tokens_mask", None) + if self.mlm: + batch["input_ids"], batch["labels"] = self.torch_mask_tokens( + batch["input_ids"], special_tokens_mask=special_tokens_mask + ) + else: + labels = batch["input_ids"].clone() + if self.tokenizer.pad_token_id is not None: + labels[labels == self.tokenizer.pad_token_id] = -100 + batch["labels"] = labels + return batch + + def torch_mask_tokens(self, inputs: Any, special_tokens_mask: Optional[Any] = None) -> Tuple[Any, Any]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. + """ + import torch + + labels = inputs.clone() + # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`) + probability_matrix = torch.full(labels.shape, self.mlm_probability) + if special_tokens_mask is None: + special_tokens_mask = [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist() + ] + special_tokens_mask = torch.tensor(special_tokens_mask, dtype=torch.bool) + else: + special_tokens_mask = special_tokens_mask.bool() + + probability_matrix.masked_fill_(special_tokens_mask, value=0.0) + masked_indices = torch.bernoulli(probability_matrix, generator=self.generator).bool() + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + # mask_replace_prob% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = ( + torch.bernoulli(torch.full(labels.shape, self.mask_replace_prob), generator=self.generator).bool() + & masked_indices + ) + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token) + + if self.mask_replace_prob == 1 or self.random_replace_prob == 0: + return inputs, labels + + remaining_prob = 1 - self.mask_replace_prob + # scaling the random_replace_prob to the remaining probability for example if + # mask_replace_prob = 0.8 and random_replace_prob = 0.1, + # then random_replace_prob_scaled = 0.1 / 0.2 = 0.5 + random_replace_prob_scaled = self.random_replace_prob / remaining_prob + + # random_replace_prob% of the time, we replace masked input tokens with random word + indices_random = ( + torch.bernoulli(torch.full(labels.shape, random_replace_prob_scaled), generator=self.generator).bool() + & masked_indices + & ~indices_replaced + ) + random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long, generator=self.generator) + inputs[indices_random] = random_words[indices_random] + + # The rest of the time ((1-random_replace_prob-mask_replace_prob)% of the time) we keep the masked input tokens unchanged + return inputs, labels + + def numpy_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + # Handle dict or lists with proper padding and conversion to tensor. + + if self.seed and self.generator is None: + # If we have a seed, we need to create a generator object. Subsequent calls to this function will use the same generator. + # If no seed supplied, we will use the global RNG + self.create_rng() + + if isinstance(examples[0], Mapping): + batch = pad_without_fast_tokenizer_warning( + self.tokenizer, examples, return_tensors="np", pad_to_multiple_of=self.pad_to_multiple_of + ) + else: + batch = { + "input_ids": _numpy_collate_batch(examples, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + } + + # If special token mask has been preprocessed, pop it from the dict. + special_tokens_mask = batch.pop("special_tokens_mask", None) + if self.mlm: + batch["input_ids"], batch["labels"] = self.numpy_mask_tokens( + batch["input_ids"], special_tokens_mask=special_tokens_mask + ) + else: + labels = np.copy(batch["input_ids"]) + if self.tokenizer.pad_token_id is not None: + labels[labels == self.tokenizer.pad_token_id] = -100 + batch["labels"] = labels + return batch + + def numpy_mask_tokens(self, inputs: Any, special_tokens_mask: Optional[Any] = None) -> Tuple[Any, Any]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. + """ + labels = np.copy(inputs) + # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`) + probability_matrix = np.full(labels.shape, self.mlm_probability) + if special_tokens_mask is None: + special_tokens_mask = [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist() + ] + special_tokens_mask = np.array(special_tokens_mask, dtype=bool) + else: + special_tokens_mask = special_tokens_mask.astype(bool) + + probability_matrix[special_tokens_mask] = 0 + # Numpy doesn't have bernoulli, so we use a binomial with 1 trial + if self.generator: + masked_indices = self.generator.binomial(1, probability_matrix, size=probability_matrix.shape).astype(bool) + else: + masked_indices = np.random.binomial(1, probability_matrix, size=probability_matrix.shape).astype(bool) + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + # mask_replace_prob% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + if self.generator: + indices_replaced = ( + self.generator.binomial(1, self.mask_replace_prob, size=labels.shape).astype(bool) & masked_indices + ) + else: + indices_replaced = ( + np.random.binomial(1, self.mask_replace_prob, size=labels.shape).astype(bool) & masked_indices + ) + inputs[indices_replaced] = self.tokenizer.mask_token_id + + if self.mask_replace_prob == 1 or self.random_replace_prob == 0: + return inputs, labels + + remaining_prob = 1 - self.mask_replace_prob + # scaling the random_replace_prob to the remaining probability for example if + # mask_replace_prob = 0.8 and random_replace_prob = 0.1, + # then random_replace_prob_scaled = 0.1 / 0.2 = 0.5 + random_replace_prob_scaled = self.random_replace_prob / remaining_prob + if self.generator: + indices_random = ( + self.generator.binomial(1, random_replace_prob_scaled, size=labels.shape).astype(bool) + & masked_indices + & ~indices_replaced + ) + random_words = self.generator.integers( + low=0, high=len(self.tokenizer), size=np.count_nonzero(indices_random), dtype=np.int64 + ) + else: + indices_random = ( + np.random.binomial(1, random_replace_prob_scaled, size=labels.shape).astype(bool) + & masked_indices + & ~indices_replaced + ) + random_words = np.random.randint( + low=0, high=len(self.tokenizer), size=np.count_nonzero(indices_random), dtype=np.int64 + ) + inputs[indices_random] = random_words + + # The rest of the time (10% of the time) we keep the masked input tokens unchanged + return inputs, labels + + +@dataclass +class DataCollatorForWholeWordMask(DataCollatorForLanguageModeling): + """ + Data collator used for language modeling that masks entire words. + + - collates batches of tensors, honoring their tokenizer's pad_token + - preprocesses batches for masked language modeling + + + + This collator relies on details of the implementation of subword tokenization by [`BertTokenizer`], specifically + that subword tokens are prefixed with *##*. For tokenizers that do not adhere to this scheme, this collator will + produce an output that is roughly equivalent to [`.DataCollatorForLanguageModeling`]. + + """ + + def torch_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + if self.seed and self.generator is None: + # If we have a seed, we need to create a generator object. Subsequent calls to this function will use the same generator. + # If no seed supplied, we will use the global RNG + self.create_rng() + + if isinstance(examples[0], Mapping): + input_ids = [e["input_ids"] for e in examples] + else: + input_ids = examples + examples = [{"input_ids": e} for e in examples] + + batch_input = _torch_collate_batch(input_ids, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + + mask_labels = [] + for e in examples: + ref_tokens = [] + for id in tolist(e["input_ids"]): + token = self.tokenizer._convert_id_to_token(id) + ref_tokens.append(token) + + # For Chinese tokens, we need extra inf to mark sub-word, e.g [喜,欢]-> [喜，##欢] + if "chinese_ref" in e: + ref_pos = tolist(e["chinese_ref"]) + len_seq = len(e["input_ids"]) + for i in range(len_seq): + if i in ref_pos: + ref_tokens[i] = "##" + ref_tokens[i] + mask_labels.append(self._whole_word_mask(ref_tokens)) + batch_mask = _torch_collate_batch(mask_labels, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + inputs, labels = self.torch_mask_tokens(batch_input, batch_mask) + return {"input_ids": inputs, "labels": labels} + + def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + import tensorflow as tf + + if self.seed and self.generator is None: + # If we have a seed, we need to create a generator object. Subsequent calls to this function will use the same generator. + # If no seed supplied, we will use the global RNG + self.create_rng() + + if isinstance(examples[0], Mapping): + input_ids = [e["input_ids"] for e in examples] + else: + input_ids = examples + examples = [{"input_ids": e} for e in examples] + + batch_input = _tf_collate_batch(input_ids, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + + mask_labels = [] + for e in examples: + ref_tokens = [] + for id in tolist(e["input_ids"]): + token = self.tokenizer._convert_id_to_token(id) + ref_tokens.append(token) + + # For Chinese tokens, we need extra inf to mark sub-word, e.g [喜,欢]-> [喜，##欢] + if "chinese_ref" in e: + ref_pos = tolist(e["chinese_ref"]) + len_seq = len(e["input_ids"]) + for i in range(len_seq): + if i in ref_pos: + ref_tokens[i] = "##" + ref_tokens[i] + mask_labels.append(self._whole_word_mask(ref_tokens)) + batch_mask = _tf_collate_batch(mask_labels, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + inputs, labels = self.tf_mask_tokens(tf.cast(batch_input, tf.int64), batch_mask) + return {"input_ids": inputs, "labels": labels} + + def numpy_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + if self.seed and self.generator is None: + # If we have a seed, we need to create a generator object. Subsequent calls to this function will use the same generator. + # If no seed supplied, we will use the global RNG + self.create_rng() + + if isinstance(examples[0], Mapping): + input_ids = [e["input_ids"] for e in examples] + else: + input_ids = examples + examples = [{"input_ids": e} for e in examples] + + batch_input = _numpy_collate_batch(input_ids, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + + mask_labels = [] + for e in examples: + ref_tokens = [] + for id in tolist(e["input_ids"]): + token = self.tokenizer._convert_id_to_token(id) + ref_tokens.append(token) + + # For Chinese tokens, we need extra inf to mark sub-word, e.g [喜,欢]-> [喜，##欢] + if "chinese_ref" in e: + ref_pos = tolist(e["chinese_ref"]) + len_seq = len(e["input_ids"]) + for i in range(len_seq): + if i in ref_pos: + ref_tokens[i] = "##" + ref_tokens[i] + mask_labels.append(self._whole_word_mask(ref_tokens)) + batch_mask = _numpy_collate_batch(mask_labels, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) + inputs, labels = self.numpy_mask_tokens(batch_input, batch_mask) + return {"input_ids": inputs, "labels": labels} + + def _shuffle(self, cand_indexes): + # if no seed, just use random's shuffle + if self.seed is None: + random.shuffle(cand_indexes) + return cand_indexes + + # if seed is provided, use the generator to shuffle + if self.return_tensors == "pt": + import torch + + indices = torch.randperm(len(cand_indexes), generator=self.generator) + return [cand_indexes[i] for i in indices] + + elif self.return_tensors == "tf": + import tensorflow as tf + + seed = self.generator.make_seeds(2)[0] + indices = tf.random.experimental.stateless_shuffle(tf.range(len(cand_indexes)), seed=seed).numpy().tolist() + return [cand_indexes[i] for i in indices] + + elif self.return_tensors == "np": + self.generator.shuffle(cand_indexes) + return cand_indexes + + def _whole_word_mask(self, input_tokens: List[str], max_predictions=512): + """ + Get 0/1 labels for masked tokens with whole word mask proxy + """ + if not isinstance(self.tokenizer, (BertTokenizer, BertTokenizerFast)): + warnings.warn( + "DataCollatorForWholeWordMask is only suitable for BertTokenizer-like tokenizers. " + "Please refer to the documentation for more information." + ) + + cand_indexes = [] + for i, token in enumerate(input_tokens): + if token == "[CLS]" or token == "[SEP]": + continue + + if len(cand_indexes) >= 1 and token.startswith("##"): + cand_indexes[-1].append(i) + else: + cand_indexes.append([i]) + + cand_indexes = self._shuffle(cand_indexes) + num_to_predict = min(max_predictions, max(1, int(round(len(input_tokens) * self.mlm_probability)))) + masked_lms = [] + covered_indexes = set() + for index_set in cand_indexes: + if len(masked_lms) >= num_to_predict: + break + # If adding a whole-word mask would exceed the maximum number of + # predictions, then just skip this candidate. + if len(masked_lms) + len(index_set) > num_to_predict: + continue + for index in index_set: + covered_indexes.add(index) + masked_lms.append(index) + + if len(covered_indexes) != len(masked_lms): + raise ValueError("Length of covered_indexes is not equal to length of masked_lms.") + mask_labels = [1 if i in covered_indexes else 0 for i in range(len(input_tokens))] + return mask_labels + + def torch_mask_tokens(self, inputs: Any, mask_labels: Any) -> Tuple[Any, Any]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set + 'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref. + """ + import torch + + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the" + " --mlm flag if you want to use this tokenizer." + ) + labels = inputs.clone() + # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa) + + probability_matrix = mask_labels + + special_tokens_mask = [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist() + ] + probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0) + if self.tokenizer.pad_token is not None: + padding_mask = labels.eq(self.tokenizer.pad_token_id) + probability_matrix.masked_fill_(padding_mask, value=0.0) + + masked_indices = probability_matrix.bool() + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + # mask_replace_prob% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = ( + torch.bernoulli(torch.full(labels.shape, self.mask_replace_prob), generator=self.generator).bool() + & masked_indices + ) + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token) + + if self.mask_replace_prob == 1 or self.random_replace_prob == 0: + return inputs, labels + + remaining_prob = 1 - self.mask_replace_prob + # scaling the random_replace_prob to the remaining probability for example if + # mask_replace_prob = 0.8 and random_replace_prob = 0.1, + # then random_replace_prob_scaled = 0.1 / 0.2 = 0.5 + random_replace_prob_scaled = self.random_replace_prob / remaining_prob + + # random_replacement_prob% of the time, we replace masked input tokens with random word + indices_random = ( + torch.bernoulli(torch.full(labels.shape, random_replace_prob_scaled), generator=self.generator).bool() + & masked_indices + & ~indices_replaced + ) + random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long, generator=self.generator) + inputs[indices_random] = random_words[indices_random] + + # The rest of the time ((1-random_replacement_prob-mask_replace_prob)% of the time) we keep the masked input tokens unchanged + return inputs, labels + + def tf_mask_tokens(self, inputs: Any, mask_labels: Any) -> Tuple[Any, Any]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set + 'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref. + """ + import tensorflow as tf + + input_shape = tf.shape(inputs) + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the" + " --mlm flag if you want to use this tokenizer." + ) + labels = tf.identity(inputs) + # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa) + + masked_indices = tf.cast(mask_labels, tf.bool) + + special_tokens_mask = [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels + ] + masked_indices = masked_indices & ~tf.cast(special_tokens_mask, dtype=tf.bool) + if self.tokenizer.pad_token is not None: + padding_mask = inputs == self.tokenizer.pad_token_id + masked_indices = masked_indices & ~padding_mask + + # Replace unmasked indices with -100 in the labels since we only compute loss on masked tokens + labels = tf.where(masked_indices, inputs, -100) + + # mask_replace_prob% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = self.tf_bernoulli(input_shape, self.mask_replace_prob, self.generator) & masked_indices + + inputs = tf.where(indices_replaced, self.tokenizer.mask_token_id, inputs) + + if self.mask_replace_prob == 1 or self.random_replace_prob == 0: + return inputs, labels + + remaining_prob = 1 - self.mask_replace_prob + # scaling the random_replace_prob to the remaining probability for example if + # mask_replace_prob = 0.8 and random_replace_prob = 0.1, + # then random_replace_prob_scaled = 0.1 / 0.2 = 0.5 + random_replace_prob_scaled = self.random_replace_prob / remaining_prob + + # random_replace_prob% of the time, we replace masked input tokens with random word + indices_random = ( + self.tf_bernoulli(input_shape, random_replace_prob_scaled, self.generator) + & masked_indices + & ~indices_replaced + ) + + if self.generator: + random_words = self.generator.uniform(input_shape, maxval=len(self.tokenizer), dtype=tf.int64) + else: + random_words = tf.random.uniform(input_shape, maxval=len(self.tokenizer), dtype=tf.int64) + + inputs = tf.where(indices_random, random_words, inputs) + + # The rest of the time ((1-mask_replace_prob-random_replace_prob)% of the time) we keep the masked input tokens unchanged + return inputs, labels + + def numpy_mask_tokens(self, inputs: Any, mask_labels: Any) -> Tuple[Any, Any]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set + 'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref. + """ + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the" + " --mlm flag if you want to use this tokenizer." + ) + labels = np.copy(inputs) + # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa) + + masked_indices = mask_labels.astype(bool) + + special_tokens_mask = [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist() + ] + masked_indices[np.array(special_tokens_mask, dtype=bool)] = 0 + if self.tokenizer.pad_token is not None: + padding_mask = labels == self.tokenizer.pad_token_id + masked_indices[padding_mask] = 0 + + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + # mask_replacement_prob% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + if self.generator: + indices_replaced = ( + self.generator.binomial(1, self.mask_replace_prob, size=labels.shape).astype(bool) & masked_indices + ) + else: + indices_replaced = ( + np.random.binomial(1, self.mask_replace_prob, size=labels.shape).astype(bool) & masked_indices + ) + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token) + + if self.mask_replace_prob == 1 or self.random_replace_prob == 0: + return inputs, labels + + remaining_prob = 1 - self.mask_replace_prob + # scaling the random_replace_prob to the remaining probability for example if + # mask_replace_prob = 0.8 and random_replace_prob = 0.1, + # then random_replace_prob_scaled = 0.1 / 0.2 = 0.5 + random_replace_prob_scaled = self.random_replace_prob / remaining_prob + + if self.generator: + indices_random = ( + self.generator.binomial(1, random_replace_prob_scaled, size=labels.shape).astype(bool) + & masked_indices + & ~indices_replaced + ) + random_words = self.generator.integers(low=0, high=len(self.tokenizer), size=labels.shape, dtype=np.int64) + else: + indices_random = ( + np.random.binomial(1, random_replace_prob_scaled, size=labels.shape).astype(bool) + & masked_indices + & ~indices_replaced + ) + random_words = np.random.randint(low=0, high=len(self.tokenizer), size=labels.shape, dtype=np.int64) + + inputs[indices_random] = random_words[indices_random] + + # The rest of the time ((1-mask_replace_prob-random_replace_prob)% of the time) we keep the masked input tokens unchanged + return inputs, labels + + +def tolist(x): + if isinstance(x, list): + return x + elif hasattr(x, "numpy"): # Checks for TF tensors without needing the import + x = x.numpy() + return x.tolist() + + +@dataclass +class DataCollatorForSOP(DataCollatorForLanguageModeling): + """ + Data collator used for sentence order prediction task. + + - collates batches of tensors, honoring their tokenizer's pad_token + - preprocesses batches for both masked language modeling and sentence order prediction + """ + + def __init__(self, *args, **kwargs): + warnings.warn( + "DataCollatorForSOP is deprecated and will be removed in a future version, you can now use " + "DataCollatorForLanguageModeling instead.", + FutureWarning, + ) + + def __call__(self, examples: List[Dict[str, Any]]) -> Dict[str, Any]: + import torch + from torch.nn.utils.rnn import pad_sequence + + input_ids = [example["input_ids"] for example in examples] + input_ids = _torch_collate_batch(input_ids, self.tokenizer) + input_ids, labels, attention_mask = self.mask_tokens(input_ids) + + token_type_ids = [example["token_type_ids"] for example in examples] + # size of segment_ids varied because randomness, padding zero to the end as the original implementation + token_type_ids = pad_sequence(token_type_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id) + + sop_label_list = [example["sentence_order_label"] for example in examples] + sentence_order_label = torch.stack(sop_label_list) + + return { + "input_ids": input_ids, + "labels": labels, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids, + "sentence_order_label": sentence_order_label, + } + + def mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any]: + """ + Prepare masked tokens inputs/labels/attention_mask for masked language modeling: 80% MASK, 10% random, 10% + original. N-gram not applied yet. + """ + import torch + + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the" + " --mlm flag if you want to use this tokenizer." + ) + + labels = inputs.clone() + # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa) + probability_matrix = torch.full(labels.shape, self.mlm_probability) + special_tokens_mask = [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist() + ] + probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0) + if self.tokenizer.pad_token is not None: + padding_mask = labels.eq(self.tokenizer.pad_token_id) + probability_matrix.masked_fill_(padding_mask, value=0.0) + masked_indices = torch.bernoulli(probability_matrix).bool() + # probability be `1` (masked), however in albert model attention mask `0` means masked, revert the value + attention_mask = (~masked_indices).float() + if self.tokenizer.pad_token is not None: + attention_padding_mask = labels.eq(self.tokenizer.pad_token_id) + attention_mask.masked_fill_(attention_padding_mask, value=1.0) + labels[~masked_indices] = -100 # We only compute loss on masked tokens, -100 is default for CE compute + + # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token) + + # 10% of the time, we replace masked input tokens with random word + indices_random = torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices & ~indices_replaced + random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long) + inputs[indices_random] = random_words[indices_random] + + # The rest of the time (10% of the time) we keep the masked input tokens unchanged + return inputs, labels, attention_mask + + +@dataclass +class DataCollatorForPermutationLanguageModeling(DataCollatorMixin): + """ + Data collator used for permutation language modeling. + + - collates batches of tensors, honoring their tokenizer's pad_token + - preprocesses batches for permutation language modeling with procedures specific to XLNet + """ + + tokenizer: PreTrainedTokenizerBase + plm_probability: float = 1 / 6 + max_span_length: int = 5 # maximum length of a span of masked tokens + return_tensors: str = "pt" + + def torch_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + if isinstance(examples[0], Mapping): + examples = [e["input_ids"] for e in examples] + batch = _torch_collate_batch(examples, self.tokenizer) + inputs, perm_mask, target_mapping, labels = self.torch_mask_tokens(batch) + return {"input_ids": inputs, "perm_mask": perm_mask, "target_mapping": target_mapping, "labels": labels} + + def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + if isinstance(examples[0], Mapping): + examples = [e["input_ids"] for e in examples] + batch = _tf_collate_batch(examples, self.tokenizer) + inputs, perm_mask, target_mapping, labels = self.tf_mask_tokens(batch) + return {"input_ids": inputs, "perm_mask": perm_mask, "target_mapping": target_mapping, "labels": labels} + + def numpy_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + if isinstance(examples[0], Mapping): + examples = [e["input_ids"] for e in examples] + batch = _numpy_collate_batch(examples, self.tokenizer) + inputs, perm_mask, target_mapping, labels = self.numpy_mask_tokens(batch) + return {"input_ids": inputs, "perm_mask": perm_mask, "target_mapping": target_mapping, "labels": labels} + + def torch_mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any, Any]: + """ + The masked tokens to be predicted for a particular sequence are determined by the following algorithm: + + 0. Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far). + 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked) + 2. Reserve a context of length `context_length = span_length / plm_probability` to surround span to be + masked + 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - + span_length]` and mask tokens `start_index:start_index + span_length` + 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in the + sequence to be processed), repeat from Step 1. + """ + import torch + + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for permutation language modeling." + " Please add a mask token if you want to use this tokenizer." + ) + + if inputs.size(1) % 2 != 0: + raise ValueError( + "This collator requires that sequence lengths be even to create a leakage-free perm_mask. Please see" + " relevant comments in source code for details." + ) + + labels = inputs.clone() + # Creating the mask and target_mapping tensors + masked_indices = torch.full(labels.shape, 0, dtype=torch.bool) + target_mapping = torch.zeros((labels.size(0), labels.size(1), labels.size(1)), dtype=torch.float32) + + for i in range(labels.size(0)): + # Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far). + cur_len = 0 + max_len = labels.size(1) + + while cur_len < max_len: + # Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked) + span_length = torch.randint(1, self.max_span_length + 1, (1,)).item() + # Reserve a context of length `context_length = span_length / plm_probability` to surround the span to be masked + context_length = int(span_length / self.plm_probability) + # Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - span_length]` and mask tokens `start_index:start_index + span_length` + start_index = cur_len + torch.randint(context_length - span_length + 1, (1,)).item() + masked_indices[i, start_index : start_index + span_length] = 1 + # Set `cur_len = cur_len + context_length` + cur_len += context_length + + # Since we're replacing non-masked tokens with -100 in the labels tensor instead of skipping them altogether, + # the i-th predict corresponds to the i-th token. + target_mapping[i] = torch.eye(labels.size(1)) + + special_tokens_mask = torch.tensor( + [self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()], + dtype=torch.bool, + ) + masked_indices.masked_fill_(special_tokens_mask, value=0.0) + if self.tokenizer.pad_token is not None: + padding_mask = labels.eq(self.tokenizer.pad_token_id) + masked_indices.masked_fill_(padding_mask, value=0.0) + + # Mask indicating non-functional tokens, where functional tokens are [SEP], [CLS], padding, etc. + non_func_mask = ~(padding_mask | special_tokens_mask) + + inputs[masked_indices] = self.tokenizer.mask_token_id + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + perm_mask = torch.zeros((labels.size(0), labels.size(1), labels.size(1)), dtype=torch.float32) + + for i in range(labels.size(0)): + # Generate permutation indices i.e. sample a random factorisation order for the sequence. This will + # determine which tokens a given token can attend to (encoded in `perm_mask`). + # Note: Length of token sequence being permuted has to be less than or equal to reused sequence length + # (see documentation for `mems`), otherwise information may leak through due to reuse. In this implementation, + # we assume that reused length is half of sequence length and permutation length is equal to reused length. + # This requires that the sequence length be even. + + # Create a linear factorisation order + perm_index = torch.arange(labels.size(1)) + # Split this into two halves, assuming that half the sequence is reused each time + perm_index = perm_index.reshape((-1, labels.size(1) // 2)).transpose(0, 1) + # Permute the two halves such that they do not cross over + perm_index = perm_index[torch.randperm(labels.size(1) // 2)] + # Flatten this out into the desired permuted factorisation order + perm_index = torch.flatten(perm_index.transpose(0, 1)) + # Set the permutation indices of non-masked (non-functional) tokens to the + # smallest index (-1) so that: + # (1) They can be seen by all other positions + # (2) They cannot see masked positions, so there won't be information leak + perm_index.masked_fill_(~masked_indices[i] & non_func_mask[i], -1) + # The logic for whether the i-th token can attend on the j-th token based on the factorisation order: + # 0 (can attend): If perm_index[i] > perm_index[j] or j is neither masked nor a functional token + # 1 (cannot attend): If perm_index[i] <= perm_index[j] and j is either masked or a functional token + perm_mask[i] = ( + perm_index.reshape((labels.size(1), 1)) <= perm_index.reshape((1, labels.size(1))) + ) & masked_indices[i] + + return inputs.long(), perm_mask, target_mapping, labels.long() + + def tf_mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any, Any]: + """ + The masked tokens to be predicted for a particular sequence are determined by the following algorithm: + + 0. Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far). + 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked) + 2. Reserve a context of length `context_length = span_length / plm_probability` to surround span to be + masked + 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - + span_length]` and mask tokens `start_index:start_index + span_length` + 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in the + sequence to be processed), repeat from Step 1. + """ + import tensorflow as tf + + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for permutation language modeling." + " Please add a mask token if you want to use this tokenizer." + ) + + if tf.shape(inputs)[1] % 2 != 0: + raise ValueError( + "This collator requires that sequence lengths be even to create a leakage-free perm_mask. Please see" + " relevant comments in source code for details." + ) + + labels = tf.identity(inputs) + # Creating the mask and target_mapping tensors + masked_indices = np.full(labels.shape.as_list(), 0, dtype=bool) + labels_shape = tf.shape(labels) + target_mapping = np.zeros((labels_shape[0], labels_shape[1], labels_shape[1]), dtype=np.float32) + + for i in range(len(labels)): + # Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far). + cur_len = 0 + max_len = tf.shape(labels)[1] + + while cur_len < max_len: + # Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked) + span_length = randint(1, self.max_span_length + 1) + # Reserve a context of length `context_length = span_length / plm_probability` to surround the span to be masked + context_length = int(span_length / self.plm_probability) + # Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - span_length]` and mask tokens `start_index:start_index + span_length` + start_index = cur_len + randint(0, context_length - span_length + 1) + masked_indices[i, start_index : start_index + span_length] = 1 + # Set `cur_len = cur_len + context_length` + cur_len += context_length + + # Since we're replacing non-masked tokens with -100 in the labels tensor instead of skipping them altogether, + # the i-th predict corresponds to the i-th token. + target_mapping[i] = np.eye(labels_shape[1]) + masked_indices = tf.cast(tf.convert_to_tensor(masked_indices), dtype=tf.bool) + target_mapping = tf.convert_to_tensor(target_mapping) + special_tokens_mask = tf.convert_to_tensor( + [ + self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) + for val in labels.numpy().tolist() + ], + ) + special_tokens_mask = tf.cast(special_tokens_mask, dtype=tf.bool) + masked_indices = masked_indices & ~special_tokens_mask + if self.tokenizer.pad_token is not None: + padding_mask = labels == self.tokenizer.pad_token_id + masked_indices = masked_indices & ~padding_mask + + # Mask indicating non-functional tokens, where functional tokens are [SEP], [CLS], padding, etc. + non_func_mask = ~(padding_mask | special_tokens_mask) + + inputs = tf.where(masked_indices, self.tokenizer.mask_token_id, inputs) + labels = tf.where(masked_indices, labels, -100) # We only compute loss on masked tokens + + perm_mask = [] + + for i in range(len(labels)): + # Generate permutation indices i.e. sample a random factorisation order for the sequence. This will + # determine which tokens a given token can attend to (encoded in `perm_mask`). + # Note: Length of token sequence being permuted has to be less than or equal to reused sequence length + # (see documentation for `mems`), otherwise information may leak through due to reuse. In this implementation, + # we assume that reused length is half of sequence length and permutation length is equal to reused length. + # This requires that the sequence length be even. + + # Create a linear factorisation order + # tf.range is the equivalent of torch.arange + perm_index = tf.range(labels_shape[1]) + # Split this into two halves, assuming that half the sequence is reused each time + perm_index = tf.transpose(tf.reshape(perm_index, (-1, labels_shape[1] // 2))) + # Permute the two halves such that they do not cross over + perm_index = tf.random.shuffle(perm_index) # Shuffles along the first dimension + # Flatten this out into the desired permuted factorisation order + perm_index = tf.reshape(tf.transpose(perm_index), (-1,)) + # Set the permutation indices of non-masked (non-functional) tokens to the + # smallest index (-1) so that: + # (1) They can be seen by all other positions + # (2) They cannot see masked positions, so there won't be information leak + perm_index = tf.where(~masked_indices[i] & non_func_mask[i], -1, perm_index) + # The logic for whether the i-th token can attend on the j-th token based on the factorisation order: + # 0 (can attend): If perm_index[i] > perm_index[j] or j is neither masked nor a functional token + # 1 (cannot attend): If perm_index[i] <= perm_index[j] and j is either masked or a functional token + perm_mask.append( + (tf.reshape(perm_index, (labels_shape[1], 1)) <= tf.reshape(perm_index, (1, labels_shape[1]))) + & masked_indices[i] + ) + perm_mask = tf.stack(perm_mask, axis=0) + + return tf.cast(inputs, tf.int64), tf.cast(perm_mask, tf.float32), target_mapping, tf.cast(labels, tf.int64) + + def numpy_mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any, Any]: + """ + The masked tokens to be predicted for a particular sequence are determined by the following algorithm: + + 0. Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far). + 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked) + 2. Reserve a context of length `context_length = span_length / plm_probability` to surround span to be + masked + 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - + span_length]` and mask tokens `start_index:start_index + span_length` + 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in the + sequence to be processed), repeat from Step 1. + """ + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for permutation language modeling." + " Please add a mask token if you want to use this tokenizer." + ) + + if inputs.shape[1] % 2 != 0: + raise ValueError( + "This collator requires that sequence lengths be even to create a leakage-free perm_mask. Please see" + " relevant comments in source code for details." + ) + + labels = np.copy(inputs) + # Creating the mask and target_mapping tensors + masked_indices = np.full(labels.shape, 0, dtype=bool) + target_mapping = np.zeros((labels.shape[0], labels.shape[1], labels.shape[1]), dtype=np.float32) + + for i in range(labels.shape[0]): + # Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far). + cur_len = 0 + max_len = labels.shape[1] + + while cur_len < max_len: + # Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked) + span_length = randint(1, self.max_span_length + 1) + # Reserve a context of length `context_length = span_length / plm_probability` to surround the span to be masked + context_length = int(span_length / self.plm_probability) + # Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - span_length]` and mask tokens `start_index:start_index + span_length` + start_index = cur_len + randint(0, context_length - span_length + 1) + masked_indices[i, start_index : start_index + span_length] = 1 + # Set `cur_len = cur_len + context_length` + cur_len += context_length + + # Since we're replacing non-masked tokens with -100 in the labels tensor instead of skipping them altogether, + # the i-th predict corresponds to the i-th token. + target_mapping[i] = np.eye(labels.shape[1]) + + special_tokens_mask = np.array( + [self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()], + dtype=bool, + ) + masked_indices[special_tokens_mask] = 0 + if self.tokenizer.pad_token is not None: + padding_mask = labels == self.tokenizer.pad_token_id + masked_indices[padding_mask] = 0.0 + + # Mask indicating non-functional tokens, where functional tokens are [SEP], [CLS], padding, etc. + non_func_mask = ~(padding_mask | special_tokens_mask) + + inputs[masked_indices] = self.tokenizer.mask_token_id + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + perm_mask = np.zeros((labels.shape[0], labels.shape[1], labels.shape[1]), dtype=np.float32) + + for i in range(labels.shape[0]): + # Generate permutation indices i.e. sample a random factorisation order for the sequence. This will + # determine which tokens a given token can attend to (encoded in `perm_mask`). + # Note: Length of token sequence being permuted has to be less than or equal to reused sequence length + # (see documentation for `mems`), otherwise information may leak through due to reuse. In this implementation, + # we assume that reused length is half of sequence length and permutation length is equal to reused length. + # This requires that the sequence length be even. + + # Create a linear factorisation order + perm_index = np.arange(labels.shape[1]) + # Split this into two halves, assuming that half the sequence is reused each time + perm_index = perm_index.reshape((-1, labels.shape[1] // 2)).T + # Permute the two halves such that they do not cross over + np.random.shuffle(perm_index) + # Flatten this out into the desired permuted factorisation order + perm_index = perm_index.T.flatten() + # Set the permutation indices of non-masked (non-functional) tokens to the + # smallest index (-1) so that: + # (1) They can be seen by all other positions + # (2) They cannot see masked positions, so there won't be information leak + perm_index[~masked_indices[i] & non_func_mask[i]] = -1 + # The logic for whether the i-th token can attend on the j-th token based on the factorisation order: + # 0 (can attend): If perm_index[i] > perm_index[j] or j is neither masked nor a functional token + # 1 (cannot attend): If perm_index[i] <= perm_index[j] and j is either masked or a functional token + perm_mask[i] = ( + perm_index.reshape((labels.shape[1], 1)) <= perm_index.reshape((1, labels.shape[1])) + ) & masked_indices[i] + + return inputs.astype(np.int64), perm_mask, target_mapping, labels.astype(np.int64) + + +@dataclass +class DataCollatorWithFlattening(DefaultDataCollator): + """ + Data collator used for padding free approach. Does the following: + + - concatate the entire mini batch into single long sequence [1, total_tokens] + - uses `separator_id` to separate sequences within the concatenated `labels`, default value is -100 + - no padding will be added, returns `input_ids`, `labels` and `position_ids` + + + + Using `DataCollatorWithFlattening` will flatten the entire mini batch into single long sequence. + Make sure your attention computation is able to handle it! + + + """ + + def __init__(self, *args, return_position_ids=True, separator_id=-100, **kwargs): + super().__init__(*args, **kwargs) + self.return_position_ids = return_position_ids + self.separator_id = separator_id + + def __call__(self, features, return_tensors=None, separator_id=None): + if return_tensors is None: + return_tensors = self.return_tensors + if separator_id is None: + separator_id = self.separator_id + is_labels_provided = "labels" in features[0] + ret = {"input_ids": [], "labels": []} + if self.return_position_ids: + ret.update({"position_ids": []}) + for idx in range(0, len(features)): + ret["input_ids"] += features[idx]["input_ids"] + if is_labels_provided: + ret["labels"] += [separator_id] + features[idx]["labels"][1:] + else: + ret["labels"] += [separator_id] + features[idx]["input_ids"][1:] + if self.return_position_ids: + ret["position_ids"] += list(range(len(features[idx]["input_ids"]))) + return default_data_collator([ret], return_tensors) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/debug_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/debug_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..91425d9fed7a3e4626d670b615565e6707b0f870 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/debug_utils.py @@ -0,0 +1,346 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from .utils import ExplicitEnum, is_torch_available, logging + + +if is_torch_available(): + import torch + + +logger = logging.get_logger(__name__) + + +class DebugUnderflowOverflow: + """ + This debug class helps detect and understand where the model starts getting very large or very small, and more + importantly `nan` or `inf` weight and activation elements. + + There are 2 working modes: + + 1. Underflow/overflow detection (default) + 2. Specific batch absolute min/max tracing without detection + + Mode 1: Underflow/overflow detection + + To activate the underflow/overflow detection, initialize the object with the model : + + ```python + debug_overflow = DebugUnderflowOverflow(model) + ``` + + then run the training as normal and if `nan` or `inf` gets detected in at least one of the weight, input or output + elements this module will throw an exception and will print `max_frames_to_save` frames that lead to this event, + each frame reporting + + 1. the fully qualified module name plus the class name whose `forward` was run + 2. the absolute min and max value of all elements for each module weights, and the inputs and output + + For example, here is the header and the last few frames in detection report for `google/mt5-small` run in fp16 + mixed precision : + + ``` + Detected inf/nan during batch_number=0 + Last 21 forward frames: + abs min abs max metadata + [...] + encoder.block.2.layer.1.DenseReluDense.wi_0 Linear + 2.17e-07 4.50e+00 weight + 1.79e-06 4.65e+00 input[0] + 2.68e-06 3.70e+01 output + encoder.block.2.layer.1.DenseReluDense.wi_1 Linear + 8.08e-07 2.66e+01 weight + 1.79e-06 4.65e+00 input[0] + 1.27e-04 2.37e+02 output + encoder.block.2.layer.1.DenseReluDense.wo Linear + 1.01e-06 6.44e+00 weight + 0.00e+00 9.74e+03 input[0] + 3.18e-04 6.27e+04 output + encoder.block.2.layer.1.DenseReluDense T5DenseGatedGeluDense + 1.79e-06 4.65e+00 input[0] + 3.18e-04 6.27e+04 output + encoder.block.2.layer.1.dropout Dropout + 3.18e-04 6.27e+04 input[0] + 0.00e+00 inf output + ``` + + You can see here, that `T5DenseGatedGeluDense.forward` resulted in output activations, whose absolute max value was + around 62.7K, which is very close to fp16's top limit of 64K. In the next frame we have `Dropout` which + renormalizes the weights, after it zeroed some of the elements, which pushes the absolute max value to more than + 64K, and we get an overlow. + + As you can see it's the previous frames that we need to look into when the numbers start going into very large for + fp16 numbers. + + The tracking is done in a forward hook, which gets invoked immediately after `forward` has completed. + + By default the last 21 frames are printed. You can change the default to adjust for your needs. For example : + + ```python + debug_overflow = DebugUnderflowOverflow(model, max_frames_to_save=100) + ``` + + To validate that you have set up this debugging feature correctly, and you intend to use it in a training that + may take hours to complete, first run it with normal tracing enabled for one of a few batches as explained in + the next section. + + + Mode 2. Specific batch absolute min/max tracing without detection + + The second work mode is per-batch tracing with the underflow/overflow detection feature turned off. + + Let's say you want to watch the absolute min and max values for all the ingredients of each `forward` call of a + given batch, and only do that for batches 1 and 3. Then you instantiate this class as : + + ```python + debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[1, 3]) + ``` + + And now full batches 1 and 3 will be traced using the same format as explained above. Batches are 0-indexed. + + This is helpful if you know that the program starts misbehaving after a certain batch number, so you can + fast-forward right to that area. + + + Early stopping: + + You can also specify the batch number after which to stop the training, with : + + ```python + debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[1, 3], abort_after_batch_num=3) + ``` + + This feature is mainly useful in the tracing mode, but you can use it for any mode. + + + **Performance**: + + As this module measures absolute `min`/``max` of each weight of the model on every forward it'll slow the training + down. Therefore remember to turn it off once the debugging needs have been met. + + Args: + model (`nn.Module`): + The model to debug. + max_frames_to_save (`int`, *optional*, defaults to 21): + How many frames back to record + trace_batch_nums(`List[int]`, *optional*, defaults to `[]`): + Which batch numbers to trace (turns detection off) + abort_after_batch_num (`int``, *optional*): + Whether to abort after a certain batch number has finished + """ + + def __init__(self, model, max_frames_to_save=21, trace_batch_nums=[], abort_after_batch_num=None): + self.model = model + self.trace_batch_nums = trace_batch_nums + self.abort_after_batch_num = abort_after_batch_num + + # keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence + self.frames = collections.deque([], max_frames_to_save) + self.frame = [] + self.batch_number = 0 + self.total_calls = 0 + self.detected_overflow = False + self.prefix = " " + + self.analyse_model() + + self.register_forward_hook() + + def save_frame(self, frame=None): + if frame is not None: + self.expand_frame(frame) + self.frames.append("\n".join(self.frame)) + self.frame = [] # start a new frame + + def expand_frame(self, line): + self.frame.append(line) + + def trace_frames(self): + print("\n".join(self.frames)) + self.frames = [] + + def reset_saved_frames(self): + self.frames = [] + + def dump_saved_frames(self): + print(f"\nDetected inf/nan during batch_number={self.batch_number}") + print(f"Last {len(self.frames)} forward frames:") + print(f"{'abs min':8} {'abs max':8} metadata") + print("\n".join(self.frames)) + print("\n\n") + self.frames = [] + + def analyse_model(self): + # extract the fully qualified module names, to be able to report at run time. e.g.: + # encoder.block.2.layer.0.SelfAttention.o + # + # for shared weights only the first shared module name will be registered + self.module_names = {m: name for name, m in self.model.named_modules()} + # self.longest_module_name = max(len(v) for v in self.module_names.values()) + + def analyse_variable(self, var, ctx): + if torch.is_tensor(var): + self.expand_frame(get_abs_min_max(var, ctx)) + if detect_overflow(var, ctx): + self.detected_overflow = True + elif var is None: + self.expand_frame(f"{'None':>17} {ctx}") + else: + self.expand_frame(f"{'not a tensor':>17} {ctx}") + + def batch_start_frame(self): + self.expand_frame(f"\n\n{self.prefix} *** Starting batch number={self.batch_number} ***") + self.expand_frame(f"{'abs min':8} {'abs max':8} metadata") + + def batch_end_frame(self): + self.expand_frame(f"{self.prefix} *** Finished batch number={self.batch_number - 1} ***\n\n") + + def create_frame(self, module, input, output): + self.expand_frame(f"{self.prefix} {self.module_names[module]} {module.__class__.__name__}") + + # params + for name, p in module.named_parameters(recurse=False): + self.analyse_variable(p, name) + + # inputs + if isinstance(input, tuple): + for i, x in enumerate(input): + self.analyse_variable(x, f"input[{i}]") + else: + self.analyse_variable(input, "input") + + # outputs + if isinstance(output, tuple): + for i, x in enumerate(output): + # possibly a tuple of tuples + if isinstance(x, tuple): + for j, y in enumerate(x): + self.analyse_variable(y, f"output[{i}][{j}]") + else: + self.analyse_variable(x, f"output[{i}]") + else: + self.analyse_variable(output, "output") + + self.save_frame() + + def register_forward_hook(self): + self.model.apply(self._register_forward_hook) + + def _register_forward_hook(self, module): + module.register_forward_hook(self.forward_hook) + + def forward_hook(self, module, input, output): + # - input is a tuple of packed inputs (could be non-Tensors) + # - output could be a Tensor or a tuple of Tensors and non-Tensors + + last_frame_of_batch = False + + trace_mode = True if self.batch_number in self.trace_batch_nums else False + if trace_mode: + self.reset_saved_frames() + + if self.total_calls == 0: + self.batch_start_frame() + self.total_calls += 1 + + # count batch numbers - the very first forward hook of the batch will be called when the + # batch completes - i.e. it gets called very last - we know this batch has finished + if module == self.model: + self.batch_number += 1 + last_frame_of_batch = True + + self.create_frame(module, input, output) + + # if last_frame_of_batch: + # self.batch_end_frame() + + if trace_mode: + self.trace_frames() + + if last_frame_of_batch: + self.batch_start_frame() + + if self.detected_overflow and not trace_mode: + self.dump_saved_frames() + + # now we can abort, as it's pointless to continue running + raise ValueError( + "DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. " + "Please scroll up above this traceback to see the activation values prior to this event." + ) + + # abort after certain batch if requested to do so + if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num: + raise ValueError( + f"DebugUnderflowOverflow: aborting after {self.batch_number} batches due to" + f" `abort_after_batch_num={self.abort_after_batch_num}` arg" + ) + + +def get_abs_min_max(var, ctx): + abs_var = var.abs() + return f"{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}" + + +def detect_overflow(var, ctx): + """ + Report whether the tensor contains any `nan` or `inf` entries. + + This is useful for detecting overflows/underflows and best to call right after the function that did some math that + modified the tensor in question. + + This function contains a few other helper features that you can enable and tweak directly if you want to track + various other things. + + Args: + var: the tensor variable to check + ctx: the message to print as a context + + Return: + `True` if `inf` or `nan` was detected, `False` otherwise + """ + detected = False + if torch.isnan(var).any().item(): + detected = True + print(f"{ctx} has nans") + if torch.isinf(var).any().item(): + detected = True + print(f"{ctx} has infs") + + # if needed to monitor large elements can enable the following + if 0: # and detected: + n100 = var[torch.ge(var.abs(), 100)] + if n100.numel() > 0: + print(f"{ctx}: n100={n100.numel()}") + n1000 = var[torch.ge(var.abs(), 1000)] + if n1000.numel() > 0: + print(f"{ctx}: n1000={n1000.numel()}") + n10000 = var[torch.ge(var.abs(), 10000)] + if n10000.numel() > 0: + print(f"{ctx}: n10000={n10000.numel()}") + + if 0: + print(f"min={var.min():9.2e} max={var.max():9.2e}") + + if 0: + print(f"min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})") + + return detected + + +class DebugOption(ExplicitEnum): + UNDERFLOW_OVERFLOW = "underflow_overflow" + TPU_METRICS_DEBUG = "tpu_metrics_debug" diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/deepspeed.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/deepspeed.py new file mode 100644 index 0000000000000000000000000000000000000000..1700301db51fd73bd9023f68170066d88c6971eb --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/deepspeed.py @@ -0,0 +1,498 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Integration with Deepspeed +""" + +import copy +import importlib.metadata as importlib_metadata +import importlib.util +import weakref +from functools import partialmethod + +from ..dependency_versions_check import dep_version_check +from ..utils import is_accelerate_available, is_torch_available, logging + + +if is_torch_available(): + import torch + from torch import nn + + +logger = logging.get_logger(__name__) + + +def is_deepspeed_available(): + package_exists = importlib.util.find_spec("deepspeed") is not None + + # Check we're not importing a "deepspeed" directory somewhere but the actual library by trying to grab the version + # AND checking it has an author field in the metadata that is HuggingFace. + if package_exists: + try: + _ = importlib_metadata.metadata("deepspeed") + return True + except importlib_metadata.PackageNotFoundError: + return False + + +if is_accelerate_available() and is_deepspeed_available(): + from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig +else: + # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file. + # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available. + from builtins import object as DeepSpeedConfig + + +class HfDeepSpeedConfig(DeepSpeedConfig): + """ + This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage. + + A `weakref` of this object is stored in the module's globals to be able to access the config from areas where + things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore + it's important that this object remains alive while the program is still running. + + [`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration + with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic + the DeepSpeed configuration is not modified in any way. + + Args: + config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict. + + """ + + def __init__(self, config_file_or_dict): + # set global weakref object + set_hf_deepspeed_config(self) + dep_version_check("accelerate") + dep_version_check("deepspeed") + super().__init__(config_file_or_dict) + + +class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig): + """ + The `HfTrainerDeepSpeedConfig` object is meant to be created during `TrainingArguments` object creation and has the + same lifespan as the latter. + """ + + def __init__(self, config_file_or_dict): + super().__init__(config_file_or_dict) + self._dtype = None + self.mismatches = [] + + def dtype(self): + if self._dtype is None: + raise ValueError("trainer_config_process() wasn't called yet to tell dtype") + return self._dtype + + def is_auto(self, ds_key_long): + val = self.get_value(ds_key_long) + if val is None: + return False + else: + return val == "auto" + + def fill_match(self, ds_key_long, hf_val, hf_key=None, must_match=True): + """ + A utility method that massages the config file and can optionally verify that the values match. + + 1. Replace "auto" values with `TrainingArguments` value. + + 2. If it wasn't "auto" and `must_match` is true, then check that DS config matches Trainer + config values and if mismatched add the entry to `self.mismatched` - will assert during + `trainer_config_finalize` for one or more mismatches. + + """ + config, ds_key = self.find_config_node(ds_key_long) + if config is None: + return + + if config.get(ds_key) == "auto": + config[ds_key] = hf_val + return + + if not must_match: + return + + ds_val = config.get(ds_key) + if ds_val is not None and ds_val != hf_val: + self.mismatches.append(f"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}") + + fill_only = partialmethod(fill_match, must_match=False) + + def trainer_config_process(self, args, auto_find_batch_size=False): + """ + Adjust the config with `TrainingArguments` values. This stage is run during `TrainingArguments` object + creation. + """ + # DeepSpeed does: + # train_batch_size = world_size * train_micro_batch_size_per_gpu * gradient_accumulation_steps + train_batch_size = args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps + self.fill_match( + "train_micro_batch_size_per_gpu", + args.per_device_train_batch_size, + "per_device_train_batch_size", + not auto_find_batch_size, + ) + self.fill_match( + "gradient_accumulation_steps", + args.gradient_accumulation_steps, + "gradient_accumulation_steps", + ) + self.fill_match( + "train_batch_size", + train_batch_size, + "train_batch_size (calculated)", + not auto_find_batch_size, + ) + self.fill_match("gradient_clipping", args.max_grad_norm, "max_grad_norm") + + self.fill_match("optimizer.params.lr", args.learning_rate, "learning_rate") + self.fill_match( + "optimizer.params.betas", + [args.adam_beta1, args.adam_beta2], + "adam_beta1+adam_beta2", + ) + self.fill_match("optimizer.params.eps", args.adam_epsilon, "adam_epsilon") + self.fill_match("optimizer.params.weight_decay", args.weight_decay, "weight_decay") + + self.fill_only("scheduler.params.warmup_min_lr", 0) # not a trainer arg + self.fill_match("scheduler.params.warmup_max_lr", args.learning_rate, "learning_rate") + # total_num_steps - will get set in trainer_config_finalize + + # fp16 + if args.fp16 or args.fp16_full_eval: + fp16_backend = "apex" if args.fp16_backend == "apex" else "amp" + else: + fp16_backend = None + + if args.save_on_each_node: + # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True + self.config["checkpoint"] = self.config.get("checkpoint", {}) + self.config["checkpoint"]["use_node_local_storage"] = args.save_on_each_node + + # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set + # any here unless the user did the work + self.fill_match( + "fp16.enabled", + ((args.fp16 or args.fp16_full_eval) and fp16_backend == "amp"), + "fp16|fp16_full_eval+fp16_backend(amp)", + ) + + # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any + # ZeRO features + self.fill_match("amp.enabled", fp16_backend == "apex", "fp16+fp16_backend(apex)") + self.fill_match("amp.opt_level", args.fp16_opt_level, "fp16_opt_level") + + self.fill_match("bf16.enabled", (args.bf16 or args.bf16_full_eval), "bf16|bf16_full_eval") + + # deepspeed's default mode is fp16 unless there is a config that says differently + if self.is_true("bf16.enabled"): + self._dtype = torch.bfloat16 + elif self.is_false("fp16.enabled"): + self._dtype = torch.float32 + else: + self._dtype = torch.float16 + + def trainer_config_finalize(self, args, model, num_training_steps): + """ + This stage is run after we have the model and know num_training_steps. + + Now we can complete the configuration process. + """ + # zero + + # deal with config keys that use `auto` value and rely on model's hidden_size + hidden_size_based_keys = [ + "zero_optimization.reduce_bucket_size", + "zero_optimization.stage3_prefetch_bucket_size", + "zero_optimization.stage3_param_persistence_threshold", + ] + hidden_size_auto_keys = [x for x in hidden_size_based_keys if self.is_auto(x)] + + if len(hidden_size_auto_keys) > 0: + if hasattr(model.config, "hidden_size"): + hidden_size = model.config.hidden_size + elif hasattr(model.config, "hidden_sizes"): + # if there are many hidden sizes pick the largest one + hidden_size = max(model.config.hidden_sizes) + elif hasattr(model.config, "text_config") and hasattr(model.config.text_config, "hidden_size"): + hidden_size = model.config.text_config.hidden_size + elif hasattr(model.config, "text_config") and hasattr(model.config.text_config, "hidden_sizes"): + # if there are many hidden sizes pick the largest one + hidden_size = max(model.config.text_config.hidden_sizes) + else: + raise ValueError( + "The model's config file has neither `hidden_size` nor `hidden_sizes` entry, " + "therefore it's not possible to automatically fill out the following `auto` entries " + f"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing " + "`auto` values for these keys with an integer value of your choice." + ) + + self.fill_only("zero_optimization.reduce_bucket_size", hidden_size * hidden_size) + if self.is_zero3(): + # automatically assign the optimal config values based on model config + self.fill_only( + "zero_optimization.stage3_prefetch_bucket_size", + int(0.9 * hidden_size * hidden_size), + ) + self.fill_only( + "zero_optimization.stage3_param_persistence_threshold", + 10 * hidden_size, + ) + + # scheduler + self.fill_match( + "scheduler.params.total_num_steps", + num_training_steps, + "num_training_steps (calculated)", + ) + self.fill_match( + "scheduler.params.warmup_num_steps", + args.get_warmup_steps(num_training_steps), + "warmup_steps", + ) + + if len(self.mismatches) > 0: + mismatches = "\n".join(self.mismatches) + raise ValueError( + "Please correct the following DeepSpeed config values that mismatch TrainingArguments" + f" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'." + ) + + +# keep the config object global to be able to access it anywhere during TrainingArguments life-cycle +_hf_deepspeed_config_weak_ref = None + + +def set_hf_deepspeed_config(hf_deepspeed_config_obj): + # this is a special weakref global object to allow us to get to Deepspeed config from APIs + # that don't have an easy way to get to the Deepspeed config outside of the Trainer domain. + global _hf_deepspeed_config_weak_ref + # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed) + _hf_deepspeed_config_weak_ref = weakref.ref(hf_deepspeed_config_obj) + + +def unset_hf_deepspeed_config(): + # useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method + global _hf_deepspeed_config_weak_ref + _hf_deepspeed_config_weak_ref = None + + +def is_deepspeed_zero3_enabled(): + if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None: + return _hf_deepspeed_config_weak_ref().is_zero3() + else: + return False + + +def deepspeed_config(): + if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None: + return _hf_deepspeed_config_weak_ref().config + else: + return None + + +def _load_state_dict_into_zero3_model(model_to_load, state_dict): + """ + Loads state dict into a model specifically for Zero3, since DeepSpeed does not support the `transformers` + tensor parallelism API. + + Nearly identical code to PyTorch's `_load_from_state_dict` + """ + # copy state_dict so `_load_state_dict_into_zero3_model` can modify it + metadata = getattr(state_dict, "_metadata", None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + error_msgs = [] + + # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants + # so we need to apply the function recursively. + def load(module: nn.Module, state_dict, prefix="", assign_to_params_buffers=False): + local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) + local_metadata["assign_to_params_buffers"] = assign_to_params_buffers + + args = (state_dict, prefix, local_metadata, True, [], [], error_msgs) + # Parameters of module and children will start with prefix. We can exit early if there are none in this + # state_dict + if is_deepspeed_zero3_enabled() and len([key for key in state_dict if key.startswith(prefix)]) > 0: + import deepspeed + + # In sharded models, each shard has only part of the full state_dict, so only gather + # parameters that are in the current state_dict. + named_parameters = dict(module.named_parameters(prefix=prefix[:-1], recurse=False)) + params_to_gather = [named_parameters[k] for k in state_dict.keys() if k in named_parameters] + if len(params_to_gather) > 0: + # because zero3 puts placeholders in model params, this context + # manager gathers (unpartitions) the params of the current layer, then loads from + # the state dict and then re-partitions them again + with deepspeed.zero.GatheredParameters(params_to_gather, modifier_rank=0): + if torch.distributed.get_rank() == 0: + module._load_from_state_dict(*args) + + for name, child in module._modules.items(): + if child is not None: + load(child, state_dict, prefix + name + ".", assign_to_params_buffers) + + load(model_to_load, state_dict, assign_to_params_buffers=False) + + return error_msgs + + +def deepspeed_optim_sched(trainer, hf_deepspeed_config, args, num_training_steps, model_parameters): + """ + A convenience wrapper that deals with optimizer and lr scheduler configuration. + """ + from accelerate.utils import DummyOptim, DummyScheduler + + config = hf_deepspeed_config.config + + # Mixing and matching DS schedulers and optimizers is supported unless Offload is enabled in which case it's: + # 1. DS scheduler + DS optimizer: Yes + # 2. HF scheduler + HF optimizer: Mostly* + # 3. DS scheduler + HF optimizer: Mostly* + # 4. HF scheduler + DS optimizer: Yes + # + # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB) + + optimizer = None + if "optimizer" in config: + if args.adafactor: + raise ValueError( + "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. " + "Only one optimizer can be configured." + ) + optimizer = DummyOptim(params=model_parameters) + else: + if hf_deepspeed_config.is_offload(): + logger.info( + "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the" + " custom optimizer has both CPU and GPU implementation (except LAMB)" + ) + + # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch. + # But trainer uses AdamW by default. + optimizer = trainer.create_optimizer() + # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer` + config["zero_allow_untested_optimizer"] = True + + lr_scheduler = None + if "scheduler" in config: + lr_scheduler = DummyScheduler(optimizer) + else: + if isinstance(optimizer, DummyOptim): + + def _lr_scheduler_callable(optimizer): + # create a shallow copy first, so later modifications do not affect original trainer + trainer_copy = copy.copy(trainer) + # at the time _lr_scheduler_callable is called, trainer.lr_scheduler has been set + # update it to None so that we can re-create a new scheduler + trainer_copy.lr_scheduler = None + lr_scheduler = trainer_copy.create_scheduler( + num_training_steps=num_training_steps, optimizer=optimizer + ) + return lr_scheduler + + lr_scheduler = DummyScheduler(optimizer, lr_scheduler_callable=_lr_scheduler_callable) + else: + lr_scheduler = trainer.create_scheduler(num_training_steps=num_training_steps, optimizer=optimizer) + + return optimizer, lr_scheduler + + +def deepspeed_init(trainer, num_training_steps, inference=False): + """ + Init DeepSpeed, after updating the DeepSpeed configuration with any relevant Trainer's args. + + If `resume_from_checkpoint` was passed then an attempt to resume from a previously saved checkpoint will be made. + + Args: + trainer: Trainer object + num_training_steps: per single gpu + resume_from_checkpoint: path to a checkpoint if to resume from after normal DeepSpeedEngine load + inference: launch in inference mode (no optimizer and no lr scheduler) + auto_find_batch_size: whether to ignore the `train_micro_batch_size_per_gpu` argument as it's being + set automatically by the auto batch size finder + + Returns: optimizer, lr_scheduler + + We may use `deepspeed_init` more than once during the life of Trainer, when we do - it's a temp hack based on: + https://github.com/deepspeedai/DeepSpeed/issues/1394#issuecomment-937405374 until Deepspeed fixes a bug where it + can't resume from a checkpoint after it did some stepping https://github.com/deepspeedai/DeepSpeed/issues/1612 + + """ + from deepspeed.utils import logger as ds_logger + + model = trainer.model + args = trainer.args + + hf_deepspeed_config = trainer.accelerator.state.deepspeed_plugin.hf_ds_config + + # resume config update - some bits like `model` and `num_training_steps` only become available during train + hf_deepspeed_config.trainer_config_finalize(args, model, num_training_steps) + + # set the Deepspeed log level consistent with the Trainer + ds_logger.setLevel(args.get_process_log_level()) + + if inference: + # only Z3 makes sense for the inference + if not hf_deepspeed_config.is_zero3(): + raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config") + + # in case the training config is re-used for inference + hf_deepspeed_config.del_config_sub_tree("optimizer") + hf_deepspeed_config.del_config_sub_tree("lr_scheduler") + optimizer, lr_scheduler = None, None + model_parameters = None + else: + trainer.optimizer = None # important for when deepspeed_init is used as re-init + tp_size = hf_deepspeed_config.config.get("tensor_parallel", {}).get("autotp_size", 0) + if tp_size > 1: + import deepspeed + + model = deepspeed.tp_model_init(model=model, tp_size=tp_size, dtype=hf_deepspeed_config.dtype()) + model_parameters = list(filter(lambda p: p.requires_grad, model.parameters())) + optimizer, lr_scheduler = deepspeed_optim_sched( + trainer, hf_deepspeed_config, args, num_training_steps, model_parameters + ) + + # keep for quick debug: + # from pprint import pprint; pprint(config) + + return optimizer, lr_scheduler + + +def deepspeed_load_checkpoint(deepspeed_engine, checkpoint_path, load_module_strict=True): + # it's possible that the user is trying to resume from model_path, which doesn't necessarily + # contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's + # a resume from a checkpoint and not just a local pretrained weight. So we check here if the + # path contains what looks like a deepspeed checkpoint + import glob + + deepspeed_checkpoint_dirs = sorted(glob.glob(f"{checkpoint_path}/global_step*")) + + if len(deepspeed_checkpoint_dirs) > 0: + logger.info(f"Attempting to resume from {checkpoint_path}") + # this magically updates self.optimizer and self.lr_scheduler + load_path, _ = deepspeed_engine.load_checkpoint( + checkpoint_path, + load_module_strict=load_module_strict, + load_optimizer_states=True, + load_lr_scheduler_states=True, + ) + if load_path is None: + raise ValueError(f"[deepspeed] failed to resume from checkpoint {checkpoint_path}") + else: + raise ValueError(f"Can't find a valid checkpoint at {checkpoint_path}") diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dependency_versions_check.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dependency_versions_check.py new file mode 100644 index 0000000000000000000000000000000000000000..82d07850847ec357f36ff51088ddec36aceff093 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dependency_versions_check.py @@ -0,0 +1,63 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .dependency_versions_table import deps +from .utils.versions import require_version, require_version_core + + +# define which module versions we always want to check at run time +# (usually the ones defined in `install_requires` in setup.py) +# +# order specific notes: +# - tqdm must be checked before tokenizers + +pkgs_to_check_at_runtime = [ + "python", + "tqdm", + "regex", + "requests", + "packaging", + "filelock", + "numpy", + "tokenizers", + "huggingface-hub", + "safetensors", + "accelerate", + "pyyaml", +] + +for pkg in pkgs_to_check_at_runtime: + if pkg in deps: + if pkg == "tokenizers": + # must be loaded here, or else tqdm check may fail + from .utils import is_tokenizers_available + + if not is_tokenizers_available(): + continue # not required, check version only if installed + elif pkg == "accelerate": + # must be loaded here, or else tqdm check may fail + from .utils import is_accelerate_available + + # Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of + # Transformers with PyTorch + if not is_accelerate_available(): + continue # not required, check version only if installed + + require_version_core(deps[pkg]) + else: + raise ValueError(f"can't find {pkg} in {deps.keys()}, check dependency_versions_table.py") + + +def dep_version_check(pkg, hint=None): + require_version(deps[pkg], hint) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dependency_versions_table.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dependency_versions_table.py new file mode 100644 index 0000000000000000000000000000000000000000..86cc5debec712fb82446d85cac7a7ec063644fe7 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dependency_versions_table.py @@ -0,0 +1,107 @@ +# THIS FILE HAS BEEN AUTOGENERATED. To update: +# 1. modify the `_deps` dict in setup.py +# 2. run `make deps_table_update`` +deps = { + "Pillow": "Pillow>=10.0.1,<=15.0", + "accelerate": "accelerate>=0.26.0", + "av": "av", + "beautifulsoup4": "beautifulsoup4", + "blobfile": "blobfile", + "codecarbon": "codecarbon>=2.8.1", + "cookiecutter": "cookiecutter==1.7.3", + "dataclasses": "dataclasses", + "datasets": "datasets!=2.5.0", + "deepspeed": "deepspeed>=0.9.3", + "diffusers": "diffusers", + "dill": "dill<0.3.5", + "evaluate": "evaluate>=0.2.0", + "faiss-cpu": "faiss-cpu", + "fastapi": "fastapi", + "filelock": "filelock", + "flax": "flax>=0.4.1,<=0.7.0", + "fsspec": "fsspec<2023.10.0", + "ftfy": "ftfy", + "fugashi": "fugashi>=1.0", + "GitPython": "GitPython<3.1.19", + "hf-doc-builder": "hf-doc-builder>=0.3.0", + "hf_xet": "hf_xet", + "huggingface-hub": "huggingface-hub>=0.30.0,<1.0", + "importlib_metadata": "importlib_metadata", + "ipadic": "ipadic>=1.0.0,<2.0", + "isort": "isort>=5.5.4", + "jax": "jax>=0.4.1,<=0.4.13", + "jaxlib": "jaxlib>=0.4.1,<=0.4.13", + "jieba": "jieba", + "jinja2": "jinja2>=3.1.0", + "kenlm@git+https://github.com/ydshieh/kenlm@78f664fb3dafe1468d868d71faf19534530698d5": "kenlm@git+https://github.com/ydshieh/kenlm@78f664fb3dafe1468d868d71faf19534530698d5", + "keras": "keras>2.9,<2.16", + "keras-nlp": "keras-nlp>=0.3.1,<0.14.0", + "kernels": "kernels>=0.3.2,<0.4", + "librosa": "librosa", + "natten": "natten>=0.14.6,<0.15.0", + "nltk": "nltk<=3.8.1", + "num2words": "num2words", + "numpy": "numpy>=1.17", + "onnxconverter-common": "onnxconverter-common", + "onnxruntime-tools": "onnxruntime-tools>=1.4.2", + "onnxruntime": "onnxruntime>=1.4.0", + "opencv-python": "opencv-python", + "optimum-benchmark": "optimum-benchmark>=0.3.0", + "optuna": "optuna", + "optax": "optax>=0.0.8,<=0.1.4", + "packaging": "packaging>=20.0", + "parameterized": "parameterized", + "phonemizer": "phonemizer", + "protobuf": "protobuf", + "psutil": "psutil", + "pyyaml": "pyyaml>=5.1", + "pydantic": "pydantic", + "pytest": "pytest>=7.2.0,<8.0.0", + "pytest-asyncio": "pytest-asyncio", + "pytest-rerunfailures": "pytest-rerunfailures", + "pytest-timeout": "pytest-timeout", + "pytest-xdist": "pytest-xdist", + "pytest-order": "pytest-order", + "python": "python>=3.9.0", + "ray[tune]": "ray[tune]>=2.7.0", + "regex": "regex!=2019.12.17", + "requests": "requests", + "rhoknp": "rhoknp>=1.1.0,<1.3.1", + "rjieba": "rjieba", + "rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1", + "ruff": "ruff==0.11.2", + "sacrebleu": "sacrebleu>=1.4.12,<2.0.0", + "sacremoses": "sacremoses", + "safetensors": "safetensors>=0.4.3", + "sagemaker": "sagemaker>=2.31.0", + "schedulefree": "schedulefree>=1.2.6", + "scikit-learn": "scikit-learn", + "scipy": "scipy<1.13.0", + "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92", + "sigopt": "sigopt", + "starlette": "starlette", + "sudachipy": "sudachipy>=0.6.6", + "sudachidict_core": "sudachidict_core>=20220729", + "tensorboard": "tensorboard", + "tensorflow-cpu": "tensorflow-cpu>2.9,<2.16", + "tensorflow": "tensorflow>2.9,<2.16", + "tensorflow-text": "tensorflow-text<2.16", + "tensorflow-probability": "tensorflow-probability<0.24", + "tf2onnx": "tf2onnx", + "timeout-decorator": "timeout-decorator", + "tiktoken": "tiktoken", + "timm": "timm<=1.0.11", + "tokenizers": "tokenizers>=0.21,<0.22", + "torch": "torch>=2.0", + "torchaudio": "torchaudio", + "torchvision": "torchvision", + "pyctcdecode": "pyctcdecode>=0.4.0", + "tqdm": "tqdm>=4.27", + "unidic": "unidic>=1.0.2", + "unidic_lite": "unidic_lite>=1.0.7", + "urllib3": "urllib3<2.0.0", + "uvicorn": "uvicorn", + "pytest-rich": "pytest-rich", + "libcst": "libcst", + "rich": "rich", +} diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/deprecation.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/deprecation.py new file mode 100644 index 0000000000000000000000000000000000000000..064decb14dba8f7a0a9eb860b450823c425ca43a --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/deprecation.py @@ -0,0 +1,176 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import warnings +from functools import wraps +from typing import Optional + +import packaging.version + +from .. import __version__ +from . import ExplicitEnum, is_torch_available, is_torchdynamo_compiling + + +# This is needed in case we deprecate a kwarg of a function/method being compiled +if is_torch_available(): + import torch # noqa: F401 + + +class Action(ExplicitEnum): + NONE = "none" + NOTIFY = "notify" + NOTIFY_ALWAYS = "notify_always" + RAISE = "raise" + + +def deprecate_kwarg( + old_name: str, + version: str, + new_name: Optional[str] = None, + warn_if_greater_or_equal_version: bool = False, + raise_if_greater_or_equal_version: bool = False, + raise_if_both_names: bool = False, + additional_message: Optional[str] = None, +): + """ + Function or method decorator to notify users about deprecated keyword arguments, replacing them with a new name if specified. + Note that is decorator is `torch.compile`-safe, i.e. it will not cause graph breaks (but no warning will be displayed if compiling). + + This decorator allows you to: + - Notify users when a keyword argument is deprecated. + - Automatically replace deprecated keyword arguments with new ones. + - Raise an error if deprecated arguments are used, depending on the specified conditions. + + By default, the decorator notifies the user about the deprecated argument while the `transformers.__version__` < specified `version` + in the decorator. To keep notifications with any version `warn_if_greater_or_equal_version=True` can be set. + + Parameters: + old_name (`str`): + Name of the deprecated keyword argument. + version (`str`): + The version in which the keyword argument was (or will be) deprecated. + new_name (`Optional[str]`, *optional*): + The new name for the deprecated keyword argument. If specified, the deprecated keyword argument will be replaced with this new name. + warn_if_greater_or_equal_version (`bool`, *optional*, defaults to `False`): + Whether to show warning if current `transformers` version is greater or equal to the deprecated version. + raise_if_greater_or_equal_version (`bool`, *optional*, defaults to `False`): + Whether to raise `ValueError` if current `transformers` version is greater or equal to the deprecated version. + raise_if_both_names (`bool`, *optional*, defaults to `False`): + Whether to raise `ValueError` if both deprecated and new keyword arguments are set. + additional_message (`Optional[str]`, *optional*): + An additional message to append to the default deprecation message. + + Raises: + ValueError: + If raise_if_greater_or_equal_version is True and the current version is greater than or equal to the deprecated version, or if raise_if_both_names is True and both old and new keyword arguments are provided. + + Returns: + Callable: + A wrapped function that handles the deprecated keyword arguments according to the specified parameters. + + Example usage with renaming argument: + + ```python + @deprecate_kwarg("reduce_labels", new_name="do_reduce_labels", version="6.0.0") + def my_function(do_reduce_labels): + print(do_reduce_labels) + + my_function(reduce_labels=True) # Will show a deprecation warning and use do_reduce_labels=True + ``` + + Example usage without renaming argument: + + ```python + @deprecate_kwarg("max_size", version="6.0.0") + def my_function(max_size): + print(max_size) + + my_function(max_size=1333) # Will show a deprecation warning + ``` + + """ + + deprecated_version = packaging.version.parse(version) + current_version = packaging.version.parse(__version__) + is_greater_or_equal_version = current_version >= deprecated_version + + if is_greater_or_equal_version: + version_message = f"and removed starting from version {version}" + else: + version_message = f"and will be removed in version {version}" + + def wrapper(func): + # Required for better warning message + sig = inspect.signature(func) + function_named_args = set(sig.parameters.keys()) + is_instance_method = "self" in function_named_args + is_class_method = "cls" in function_named_args + + @wraps(func) + def wrapped_func(*args, **kwargs): + # Get class + function name (just for better warning message) + func_name = func.__name__ + if is_instance_method: + func_name = f"{args[0].__class__.__name__}.{func_name}" + elif is_class_method: + func_name = f"{args[0].__name__}.{func_name}" + + minimum_action = Action.NONE + message = None + + # deprecated kwarg and its new version are set for function call -> replace it with new name + if old_name in kwargs and new_name in kwargs: + minimum_action = Action.RAISE if raise_if_both_names else Action.NOTIFY_ALWAYS + message = f"Both `{old_name}` and `{new_name}` are set for `{func_name}`. Using `{new_name}={kwargs[new_name]}` and ignoring deprecated `{old_name}={kwargs[old_name]}`." + kwargs.pop(old_name) + + # only deprecated kwarg is set for function call -> replace it with new name + elif old_name in kwargs and new_name is not None and new_name not in kwargs: + minimum_action = Action.NOTIFY + message = f"`{old_name}` is deprecated {version_message} for `{func_name}`. Use `{new_name}` instead." + kwargs[new_name] = kwargs.pop(old_name) + + # deprecated kwarg is not set for function call and new name is not specified -> just notify + elif old_name in kwargs: + minimum_action = Action.NOTIFY + message = f"`{old_name}` is deprecated {version_message} for `{func_name}`." + + if message is not None and additional_message is not None: + message = f"{message} {additional_message}" + + # update minimum_action if argument is ALREADY deprecated (current version >= deprecated version) + if is_greater_or_equal_version: + # change to (NOTIFY, NOTIFY_ALWAYS) -> RAISE if specified + # in case we want to raise error for already deprecated arguments + if raise_if_greater_or_equal_version and minimum_action != Action.NONE: + minimum_action = Action.RAISE + + # change to NOTIFY -> NONE if specified (NOTIFY_ALWAYS can't be changed to NONE) + # in case we want to ignore notifications for already deprecated arguments + elif not warn_if_greater_or_equal_version and minimum_action == Action.NOTIFY: + minimum_action = Action.NONE + + # raise error or notify user + if minimum_action == Action.RAISE: + raise ValueError(message) + # If we are compiling, we do not raise the warning as it would break compilation + elif minimum_action in (Action.NOTIFY, Action.NOTIFY_ALWAYS) and not is_torchdynamo_compiling(): + # DeprecationWarning is ignored by default, so we use FutureWarning instead + warnings.warn(message, FutureWarning, stacklevel=2) + + return func(*args, **kwargs) + + return wrapped_func + + return wrapper diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/doc.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/doc.py new file mode 100644 index 0000000000000000000000000000000000000000..f01ffc28442cd5a5dcda3ea55d3ea08eb249b54f --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/doc.py @@ -0,0 +1,1218 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Doc utilities: Utilities related to documentation +""" + +import functools +import inspect +import re +import textwrap +import types + + +def get_docstring_indentation_level(func): + """Return the indentation level of the start of the docstring of a class or function (or method).""" + # We assume classes are always defined in the global scope + if inspect.isclass(func): + return 4 + source = inspect.getsource(func) + first_line = source.splitlines()[0] + function_def_level = len(first_line) - len(first_line.lstrip()) + return 4 + function_def_level + + +def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = "".join(docstr) + (fn.__doc__ if fn.__doc__ is not None else "") + return fn + + return docstring_decorator + + +def add_start_docstrings_to_model_forward(*docstr): + def docstring_decorator(fn): + class_name = f"[`{fn.__qualname__.split('.')[0]}`]" + intro = rf""" The {class_name} forward method, overrides the `__call__` special method. + + + + Although the recipe for forward pass needs to be defined within this function, one should call the [`Module`] + instance afterwards instead of this since the former takes care of running the pre and post processing steps while + the latter silently ignores them. + + +""" + + correct_indentation = get_docstring_indentation_level(fn) + current_doc = fn.__doc__ if fn.__doc__ is not None else "" + try: + first_non_empty = next(line for line in current_doc.splitlines() if line.strip() != "") + doc_indentation = len(first_non_empty) - len(first_non_empty.lstrip()) + except StopIteration: + doc_indentation = correct_indentation + + docs = docstr + # In this case, the correct indentation level (class method, 2 Python levels) was respected, and we should + # correctly reindent everything. Otherwise, the doc uses a single indentation level + if doc_indentation == 4 + correct_indentation: + docs = [textwrap.indent(textwrap.dedent(doc), " " * correct_indentation) for doc in docstr] + intro = textwrap.indent(textwrap.dedent(intro), " " * correct_indentation) + + docstring = "".join(docs) + current_doc + fn.__doc__ = intro + docstring + return fn + + return docstring_decorator + + +def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "".join(docstr) + return fn + + return docstring_decorator + + +PT_RETURN_INTRODUCTION = r""" + Returns: + [`{full_output_type}`] or `tuple(torch.FloatTensor)`: A [`{full_output_type}`] or a tuple of + `torch.FloatTensor` (if `return_dict=False` is passed or when `config.return_dict=False`) comprising various + elements depending on the configuration ([`{config_class}`]) and inputs. + +""" + + +TF_RETURN_INTRODUCTION = r""" + Returns: + [`{full_output_type}`] or `tuple(tf.Tensor)`: A [`{full_output_type}`] or a tuple of `tf.Tensor` (if + `return_dict=False` is passed or when `config.return_dict=False`) comprising various elements depending on the + configuration ([`{config_class}`]) and inputs. + +""" + + +def _get_indent(t): + """Returns the indentation in the first line of t""" + search = re.search(r"^(\s*)\S", t) + return "" if search is None else search.groups()[0] + + +def _convert_output_args_doc(output_args_doc): + """Convert output_args_doc to display properly.""" + # Split output_arg_doc in blocks argument/description + indent = _get_indent(output_args_doc) + blocks = [] + current_block = "" + for line in output_args_doc.split("\n"): + # If the indent is the same as the beginning, the line is the name of new arg. + if _get_indent(line) == indent: + if len(current_block) > 0: + blocks.append(current_block[:-1]) + current_block = f"{line}\n" + else: + # Otherwise it's part of the description of the current arg. + # We need to remove 2 spaces to the indentation. + current_block += f"{line[2:]}\n" + blocks.append(current_block[:-1]) + + # Format each block for proper rendering + for i in range(len(blocks)): + blocks[i] = re.sub(r"^(\s+)(\S+)(\s+)", r"\1- **\2**\3", blocks[i]) + blocks[i] = re.sub(r":\s*\n\s*(\S)", r" -- \1", blocks[i]) + + return "\n".join(blocks) + + +def _prepare_output_docstrings(output_type, config_class, min_indent=None): + """ + Prepares the return part of the docstring using `output_type`. + """ + output_docstring = output_type.__doc__ + + # Remove the head of the docstring to keep the list of args only + lines = output_docstring.split("\n") + i = 0 + while i < len(lines) and re.search(r"^\s*(Args|Parameters):\s*$", lines[i]) is None: + i += 1 + if i < len(lines): + params_docstring = "\n".join(lines[(i + 1) :]) + params_docstring = _convert_output_args_doc(params_docstring) + else: + raise ValueError( + f"No `Args` or `Parameters` section is found in the docstring of `{output_type.__name__}`. Make sure it has " + "docstring and contain either `Args` or `Parameters`." + ) + + # Add the return introduction + full_output_type = f"{output_type.__module__}.{output_type.__name__}" + intro = TF_RETURN_INTRODUCTION if output_type.__name__.startswith("TF") else PT_RETURN_INTRODUCTION + intro = intro.format(full_output_type=full_output_type, config_class=config_class) + result = intro + params_docstring + + # Apply minimum indent if necessary + if min_indent is not None: + lines = result.split("\n") + # Find the indent of the first nonempty line + i = 0 + while len(lines[i]) == 0: + i += 1 + indent = len(_get_indent(lines[i])) + # If too small, add indentation to all nonempty lines + if indent < min_indent: + to_add = " " * (min_indent - indent) + lines = [(f"{to_add}{line}" if len(line) > 0 else line) for line in lines] + result = "\n".join(lines) + + return result + + +FAKE_MODEL_DISCLAIMER = """ + + + This example uses a random model as the real ones are all very big. To get proper results, you should use + {real_checkpoint} instead of {fake_checkpoint}. If you get out-of-memory when loading that checkpoint, you can try + adding `device_map="auto"` in the `from_pretrained` call. + + +""" + + +PT_TOKEN_CLASSIFICATION_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer( + ... "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="pt" + ... ) + + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> predicted_token_class_ids = logits.argmax(-1) + + >>> # Note that tokens are classified rather then input words which means that + >>> # there might be more predicted token classes than words. + >>> # Multiple token classes might account for the same word + >>> predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]] + >>> predicted_tokens_classes + {expected_output} + + >>> labels = predicted_token_class_ids + >>> loss = model(**inputs, labels=labels).loss + >>> round(loss.item(), 2) + {expected_loss} + ``` +""" + +PT_QUESTION_ANSWERING_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" + + >>> inputs = tokenizer(question, text, return_tensors="pt") + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> answer_start_index = outputs.start_logits.argmax() + >>> answer_end_index = outputs.end_logits.argmax() + + >>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1] + >>> tokenizer.decode(predict_answer_tokens, skip_special_tokens=True) + {expected_output} + + >>> # target is "nice puppet" + >>> target_start_index = torch.tensor([{qa_target_start_index}]) + >>> target_end_index = torch.tensor([{qa_target_end_index}]) + + >>> outputs = model(**inputs, start_positions=target_start_index, end_positions=target_end_index) + >>> loss = outputs.loss + >>> round(loss.item(), 2) + {expected_loss} + ``` +""" + +PT_SEQUENCE_CLASSIFICATION_SAMPLE = r""" + Example of single-label classification: + + ```python + >>> import torch + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> predicted_class_id = logits.argmax().item() + >>> model.config.id2label[predicted_class_id] + {expected_output} + + >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` + >>> num_labels = len(model.config.id2label) + >>> model = {model_class}.from_pretrained("{checkpoint}", num_labels=num_labels) + + >>> labels = torch.tensor([1]) + >>> loss = model(**inputs, labels=labels).loss + >>> round(loss.item(), 2) + {expected_loss} + ``` + + Example of multi-label classification: + + ```python + >>> import torch + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}", problem_type="multi_label_classification") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> predicted_class_ids = torch.arange(0, logits.shape[-1])[torch.sigmoid(logits).squeeze(dim=0) > 0.5] + + >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` + >>> num_labels = len(model.config.id2label) + >>> model = {model_class}.from_pretrained( + ... "{checkpoint}", num_labels=num_labels, problem_type="multi_label_classification" + ... ) + + >>> labels = torch.sum( + ... torch.nn.functional.one_hot(predicted_class_ids[None, :].clone(), num_classes=num_labels), dim=1 + ... ).to(torch.float) + >>> loss = model(**inputs, labels=labels).loss + ``` +""" + +PT_MASKED_LM_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="pt") + + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> # retrieve index of {mask} + >>> mask_token_index = (inputs.input_ids == tokenizer.mask_token_id)[0].nonzero(as_tuple=True)[0] + + >>> predicted_token_id = logits[0, mask_token_index].argmax(axis=-1) + >>> tokenizer.decode(predicted_token_id) + {expected_output} + + >>> labels = tokenizer("The capital of France is Paris.", return_tensors="pt")["input_ids"] + >>> # mask labels of non-{mask} tokens + >>> labels = torch.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100) + + >>> outputs = model(**inputs, labels=labels) + >>> round(outputs.loss.item(), 2) + {expected_loss} + ``` +""" + +PT_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + ``` +""" + +PT_MULTIPLE_CHOICE_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." + >>> choice0 = "It is eaten with a fork and a knife." + >>> choice1 = "It is eaten while held in the hand." + >>> labels = torch.tensor(0).unsqueeze(0) # choice0 is correct (according to Wikipedia ;)), batch size 1 + + >>> encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors="pt", padding=True) + >>> outputs = model(**{{k: v.unsqueeze(0) for k, v in encoding.items()}}, labels=labels) # batch size is 1 + + >>> # the linear classifier still needs to be trained + >>> loss = outputs.loss + >>> logits = outputs.logits + ``` +""" + +PT_CAUSAL_LM_SAMPLE = r""" + Example: + + ```python + >>> import torch + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs, labels=inputs["input_ids"]) + >>> loss = outputs.loss + >>> logits = outputs.logits + ``` +""" + +PT_SPEECH_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoProcessor, {model_class} + >>> import torch + >>> from datasets import load_dataset + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation", trust_remote_code=True) + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt") + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + {expected_output} + ``` +""" + +PT_SPEECH_CTC_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoProcessor, {model_class} + >>> from datasets import load_dataset + >>> import torch + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation", trust_remote_code=True) + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt") + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + >>> predicted_ids = torch.argmax(logits, dim=-1) + + >>> # transcribe speech + >>> transcription = processor.batch_decode(predicted_ids) + >>> transcription[0] + {expected_output} + + >>> inputs["labels"] = processor(text=dataset[0]["text"], return_tensors="pt").input_ids + + >>> # compute loss + >>> loss = model(**inputs).loss + >>> round(loss.item(), 2) + {expected_loss} + ``` +""" + +PT_SPEECH_SEQ_CLASS_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoFeatureExtractor, {model_class} + >>> from datasets import load_dataset + >>> import torch + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation", trust_remote_code=True) + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = feature_extractor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt") + + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> predicted_class_ids = torch.argmax(logits, dim=-1).item() + >>> predicted_label = model.config.id2label[predicted_class_ids] + >>> predicted_label + {expected_output} + + >>> # compute loss - target_label is e.g. "down" + >>> target_label = model.config.id2label[0] + >>> inputs["labels"] = torch.tensor([model.config.label2id[target_label]]) + >>> loss = model(**inputs).loss + >>> round(loss.item(), 2) + {expected_loss} + ``` +""" + + +PT_SPEECH_FRAME_CLASS_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoFeatureExtractor, {model_class} + >>> from datasets import load_dataset + >>> import torch + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation", trust_remote_code=True) + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = feature_extractor(dataset[0]["audio"]["array"], return_tensors="pt", sampling_rate=sampling_rate) + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> probabilities = torch.sigmoid(logits[0]) + >>> # labels is a one-hot array of shape (num_frames, num_speakers) + >>> labels = (probabilities > 0.5).long() + >>> labels[0].tolist() + {expected_output} + ``` +""" + + +PT_SPEECH_XVECTOR_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoFeatureExtractor, {model_class} + >>> from datasets import load_dataset + >>> import torch + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation", trust_remote_code=True) + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = feature_extractor( + ... [d["array"] for d in dataset[:2]["audio"]], sampling_rate=sampling_rate, return_tensors="pt", padding=True + ... ) + >>> with torch.no_grad(): + ... embeddings = model(**inputs).embeddings + + >>> embeddings = torch.nn.functional.normalize(embeddings, dim=-1).cpu() + + >>> # the resulting embeddings can be used for cosine similarity-based retrieval + >>> cosine_sim = torch.nn.CosineSimilarity(dim=-1) + >>> similarity = cosine_sim(embeddings[0], embeddings[1]) + >>> threshold = 0.7 # the optimal threshold is dataset-dependent + >>> if similarity < threshold: + ... print("Speakers are not the same!") + >>> round(similarity.item(), 2) + {expected_output} + ``` +""" + +PT_VISION_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoImageProcessor, {model_class} + >>> import torch + >>> from datasets import load_dataset + + >>> dataset = load_dataset("huggingface/cats-image", trust_remote_code=True) + >>> image = dataset["test"]["image"][0] + + >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = image_processor(image, return_tensors="pt") + + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + {expected_output} + ``` +""" + +PT_VISION_SEQ_CLASS_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoImageProcessor, {model_class} + >>> import torch + >>> from datasets import load_dataset + + >>> dataset = load_dataset("huggingface/cats-image", trust_remote_code=True) + >>> image = dataset["test"]["image"][0] + + >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = image_processor(image, return_tensors="pt") + + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> # model predicts one of the 1000 ImageNet classes + >>> predicted_label = logits.argmax(-1).item() + >>> print(model.config.id2label[predicted_label]) + {expected_output} + ``` +""" + + +PT_SAMPLE_DOCSTRINGS = { + "SequenceClassification": PT_SEQUENCE_CLASSIFICATION_SAMPLE, + "QuestionAnswering": PT_QUESTION_ANSWERING_SAMPLE, + "TokenClassification": PT_TOKEN_CLASSIFICATION_SAMPLE, + "MultipleChoice": PT_MULTIPLE_CHOICE_SAMPLE, + "MaskedLM": PT_MASKED_LM_SAMPLE, + "LMHead": PT_CAUSAL_LM_SAMPLE, + "BaseModel": PT_BASE_MODEL_SAMPLE, + "SpeechBaseModel": PT_SPEECH_BASE_MODEL_SAMPLE, + "CTC": PT_SPEECH_CTC_SAMPLE, + "AudioClassification": PT_SPEECH_SEQ_CLASS_SAMPLE, + "AudioFrameClassification": PT_SPEECH_FRAME_CLASS_SAMPLE, + "AudioXVector": PT_SPEECH_XVECTOR_SAMPLE, + "VisionBaseModel": PT_VISION_BASE_MODEL_SAMPLE, + "ImageClassification": PT_VISION_SEQ_CLASS_SAMPLE, +} + + +TF_TOKEN_CLASSIFICATION_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import tensorflow as tf + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer( + ... "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="tf" + ... ) + + >>> logits = model(**inputs).logits + >>> predicted_token_class_ids = tf.math.argmax(logits, axis=-1) + + >>> # Note that tokens are classified rather then input words which means that + >>> # there might be more predicted token classes than words. + >>> # Multiple token classes might account for the same word + >>> predicted_tokens_classes = [model.config.id2label[t] for t in predicted_token_class_ids[0].numpy().tolist()] + >>> predicted_tokens_classes + {expected_output} + ``` + + ```python + >>> labels = predicted_token_class_ids + >>> loss = tf.math.reduce_mean(model(**inputs, labels=labels).loss) + >>> round(float(loss), 2) + {expected_loss} + ``` +""" + +TF_QUESTION_ANSWERING_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import tensorflow as tf + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" + + >>> inputs = tokenizer(question, text, return_tensors="tf") + >>> outputs = model(**inputs) + + >>> answer_start_index = int(tf.math.argmax(outputs.start_logits, axis=-1)[0]) + >>> answer_end_index = int(tf.math.argmax(outputs.end_logits, axis=-1)[0]) + + >>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1] + >>> tokenizer.decode(predict_answer_tokens) + {expected_output} + ``` + + ```python + >>> # target is "nice puppet" + >>> target_start_index = tf.constant([{qa_target_start_index}]) + >>> target_end_index = tf.constant([{qa_target_end_index}]) + + >>> outputs = model(**inputs, start_positions=target_start_index, end_positions=target_end_index) + >>> loss = tf.math.reduce_mean(outputs.loss) + >>> round(float(loss), 2) + {expected_loss} + ``` +""" + +TF_SEQUENCE_CLASSIFICATION_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import tensorflow as tf + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf") + + >>> logits = model(**inputs).logits + + >>> predicted_class_id = int(tf.math.argmax(logits, axis=-1)[0]) + >>> model.config.id2label[predicted_class_id] + {expected_output} + ``` + + ```python + >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` + >>> num_labels = len(model.config.id2label) + >>> model = {model_class}.from_pretrained("{checkpoint}", num_labels=num_labels) + + >>> labels = tf.constant(1) + >>> loss = model(**inputs, labels=labels).loss + >>> round(float(loss), 2) + {expected_loss} + ``` +""" + +TF_MASKED_LM_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import tensorflow as tf + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="tf") + >>> logits = model(**inputs).logits + + >>> # retrieve index of {mask} + >>> mask_token_index = tf.where((inputs.input_ids == tokenizer.mask_token_id)[0]) + >>> selected_logits = tf.gather_nd(logits[0], indices=mask_token_index) + + >>> predicted_token_id = tf.math.argmax(selected_logits, axis=-1) + >>> tokenizer.decode(predicted_token_id) + {expected_output} + ``` + + ```python + >>> labels = tokenizer("The capital of France is Paris.", return_tensors="tf")["input_ids"] + >>> # mask labels of non-{mask} tokens + >>> labels = tf.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100) + + >>> outputs = model(**inputs, labels=labels) + >>> round(float(outputs.loss), 2) + {expected_loss} + ``` +""" + +TF_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import tensorflow as tf + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf") + >>> outputs = model(inputs) + + >>> last_hidden_states = outputs.last_hidden_state + ``` +""" + +TF_MULTIPLE_CHOICE_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import tensorflow as tf + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." + >>> choice0 = "It is eaten with a fork and a knife." + >>> choice1 = "It is eaten while held in the hand." + + >>> encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors="tf", padding=True) + >>> inputs = {{k: tf.expand_dims(v, 0) for k, v in encoding.items()}} + >>> outputs = model(inputs) # batch size is 1 + + >>> # the linear classifier still needs to be trained + >>> logits = outputs.logits + ``` +""" + +TF_CAUSAL_LM_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + >>> import tensorflow as tf + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf") + >>> outputs = model(inputs) + >>> logits = outputs.logits + ``` +""" + +TF_SPEECH_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoProcessor, {model_class} + >>> from datasets import load_dataset + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation", trust_remote_code=True) + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="tf") + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + {expected_output} + ``` +""" + +TF_SPEECH_CTC_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoProcessor, {model_class} + >>> from datasets import load_dataset + >>> import tensorflow as tf + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation", trust_remote_code=True) + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="tf") + >>> logits = model(**inputs).logits + >>> predicted_ids = tf.math.argmax(logits, axis=-1) + + >>> # transcribe speech + >>> transcription = processor.batch_decode(predicted_ids) + >>> transcription[0] + {expected_output} + ``` + + ```python + >>> inputs["labels"] = processor(text=dataset[0]["text"], return_tensors="tf").input_ids + + >>> # compute loss + >>> loss = model(**inputs).loss + >>> round(float(loss), 2) + {expected_loss} + ``` +""" + +TF_VISION_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoImageProcessor, {model_class} + >>> from datasets import load_dataset + + >>> dataset = load_dataset("huggingface/cats-image", trust_remote_code=True) + >>> image = dataset["test"]["image"][0] + + >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = image_processor(image, return_tensors="tf") + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + {expected_output} + ``` +""" + +TF_VISION_SEQ_CLASS_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoImageProcessor, {model_class} + >>> import tensorflow as tf + >>> from datasets import load_dataset + + >>> dataset = load_dataset("huggingface/cats-image", trust_remote_code=True) + >>> image = dataset["test"]["image"][0] + + >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = image_processor(image, return_tensors="tf") + >>> logits = model(**inputs).logits + + >>> # model predicts one of the 1000 ImageNet classes + >>> predicted_label = int(tf.math.argmax(logits, axis=-1)) + >>> print(model.config.id2label[predicted_label]) + {expected_output} + ``` +""" + +TF_SAMPLE_DOCSTRINGS = { + "SequenceClassification": TF_SEQUENCE_CLASSIFICATION_SAMPLE, + "QuestionAnswering": TF_QUESTION_ANSWERING_SAMPLE, + "TokenClassification": TF_TOKEN_CLASSIFICATION_SAMPLE, + "MultipleChoice": TF_MULTIPLE_CHOICE_SAMPLE, + "MaskedLM": TF_MASKED_LM_SAMPLE, + "LMHead": TF_CAUSAL_LM_SAMPLE, + "BaseModel": TF_BASE_MODEL_SAMPLE, + "SpeechBaseModel": TF_SPEECH_BASE_MODEL_SAMPLE, + "CTC": TF_SPEECH_CTC_SAMPLE, + "VisionBaseModel": TF_VISION_BASE_MODEL_SAMPLE, + "ImageClassification": TF_VISION_SEQ_CLASS_SAMPLE, +} + + +FLAX_TOKEN_CLASSIFICATION_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="jax") + + >>> outputs = model(**inputs) + >>> logits = outputs.logits + ``` +""" + +FLAX_QUESTION_ANSWERING_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" + >>> inputs = tokenizer(question, text, return_tensors="jax") + + >>> outputs = model(**inputs) + >>> start_scores = outputs.start_logits + >>> end_scores = outputs.end_logits + ``` +""" + +FLAX_SEQUENCE_CLASSIFICATION_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="jax") + + >>> outputs = model(**inputs) + >>> logits = outputs.logits + ``` +""" + +FLAX_MASKED_LM_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="jax") + + >>> outputs = model(**inputs) + >>> logits = outputs.logits + ``` +""" + +FLAX_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="jax") + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + ``` +""" + +FLAX_MULTIPLE_CHOICE_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." + >>> choice0 = "It is eaten with a fork and a knife." + >>> choice1 = "It is eaten while held in the hand." + + >>> encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors="jax", padding=True) + >>> outputs = model(**{{k: v[None, :] for k, v in encoding.items()}}) + + >>> logits = outputs.logits + ``` +""" + +FLAX_CAUSAL_LM_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, {model_class} + + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np") + >>> outputs = model(**inputs) + + >>> # retrieve logts for next token + >>> next_token_logits = outputs.logits[:, -1] + ``` +""" + +FLAX_SAMPLE_DOCSTRINGS = { + "SequenceClassification": FLAX_SEQUENCE_CLASSIFICATION_SAMPLE, + "QuestionAnswering": FLAX_QUESTION_ANSWERING_SAMPLE, + "TokenClassification": FLAX_TOKEN_CLASSIFICATION_SAMPLE, + "MultipleChoice": FLAX_MULTIPLE_CHOICE_SAMPLE, + "MaskedLM": FLAX_MASKED_LM_SAMPLE, + "BaseModel": FLAX_BASE_MODEL_SAMPLE, + "LMHead": FLAX_CAUSAL_LM_SAMPLE, +} + + +def filter_outputs_from_example(docstring, **kwargs): + """ + Removes the lines testing an output with the doctest syntax in a code sample when it's set to `None`. + """ + for key, value in kwargs.items(): + if value is not None: + continue + + doc_key = "{" + key + "}" + docstring = re.sub(rf"\n([^\n]+)\n\s+{doc_key}\n", "\n", docstring) + + return docstring + + +def add_code_sample_docstrings( + *docstr, + processor_class=None, + checkpoint=None, + output_type=None, + config_class=None, + mask="[MASK]", + qa_target_start_index=14, + qa_target_end_index=15, + model_cls=None, + modality=None, + expected_output=None, + expected_loss=None, + real_checkpoint=None, + revision=None, +): + def docstring_decorator(fn): + # model_class defaults to function's class if not specified otherwise + model_class = fn.__qualname__.split(".")[0] if model_cls is None else model_cls + + if model_class[:2] == "TF": + sample_docstrings = TF_SAMPLE_DOCSTRINGS + elif model_class[:4] == "Flax": + sample_docstrings = FLAX_SAMPLE_DOCSTRINGS + else: + sample_docstrings = PT_SAMPLE_DOCSTRINGS + + # putting all kwargs for docstrings in a dict to be used + # with the `.format(**doc_kwargs)`. Note that string might + # be formatted with non-existing keys, which is fine. + doc_kwargs = { + "model_class": model_class, + "processor_class": processor_class, + "checkpoint": checkpoint, + "mask": mask, + "qa_target_start_index": qa_target_start_index, + "qa_target_end_index": qa_target_end_index, + "expected_output": expected_output, + "expected_loss": expected_loss, + "real_checkpoint": real_checkpoint, + "fake_checkpoint": checkpoint, + "true": "{true}", # For syntax that conflicts with formatting. + } + + if ("SequenceClassification" in model_class or "AudioClassification" in model_class) and modality == "audio": + code_sample = sample_docstrings["AudioClassification"] + elif "SequenceClassification" in model_class: + code_sample = sample_docstrings["SequenceClassification"] + elif "QuestionAnswering" in model_class: + code_sample = sample_docstrings["QuestionAnswering"] + elif "TokenClassification" in model_class: + code_sample = sample_docstrings["TokenClassification"] + elif "MultipleChoice" in model_class: + code_sample = sample_docstrings["MultipleChoice"] + elif "MaskedLM" in model_class or model_class in ["FlaubertWithLMHeadModel", "XLMWithLMHeadModel"]: + code_sample = sample_docstrings["MaskedLM"] + elif "LMHead" in model_class or "CausalLM" in model_class: + code_sample = sample_docstrings["LMHead"] + elif "CTC" in model_class: + code_sample = sample_docstrings["CTC"] + elif "AudioFrameClassification" in model_class: + code_sample = sample_docstrings["AudioFrameClassification"] + elif "XVector" in model_class and modality == "audio": + code_sample = sample_docstrings["AudioXVector"] + elif "Model" in model_class and modality == "audio": + code_sample = sample_docstrings["SpeechBaseModel"] + elif "Model" in model_class and modality == "vision": + code_sample = sample_docstrings["VisionBaseModel"] + elif "Model" in model_class or "Encoder" in model_class: + code_sample = sample_docstrings["BaseModel"] + elif "ImageClassification" in model_class: + code_sample = sample_docstrings["ImageClassification"] + else: + raise ValueError(f"Docstring can't be built for model {model_class}") + + code_sample = filter_outputs_from_example( + code_sample, expected_output=expected_output, expected_loss=expected_loss + ) + if real_checkpoint is not None: + code_sample = FAKE_MODEL_DISCLAIMER + code_sample + func_doc = (fn.__doc__ or "") + "".join(docstr) + output_doc = "" if output_type is None else _prepare_output_docstrings(output_type, config_class) + built_doc = code_sample.format(**doc_kwargs) + if revision is not None: + if re.match(r"^refs/pr/\\d+", revision): + raise ValueError( + f"The provided revision '{revision}' is incorrect. It should point to" + " a pull request reference on the hub like 'refs/pr/6'" + ) + built_doc = built_doc.replace( + f'from_pretrained("{checkpoint}")', f'from_pretrained("{checkpoint}", revision="{revision}")' + ) + + fn.__doc__ = func_doc + output_doc + built_doc + return fn + + return docstring_decorator + + +def replace_return_docstrings(output_type=None, config_class=None): + def docstring_decorator(fn): + func_doc = fn.__doc__ + lines = func_doc.split("\n") + i = 0 + while i < len(lines) and re.search(r"^\s*Returns?:\s*$", lines[i]) is None: + i += 1 + if i < len(lines): + indent = len(_get_indent(lines[i])) + lines[i] = _prepare_output_docstrings(output_type, config_class, min_indent=indent) + func_doc = "\n".join(lines) + else: + raise ValueError( + f"The function {fn} should have an empty 'Return:' or 'Returns:' in its docstring as placeholder, " + f"current docstring is:\n{func_doc}" + ) + fn.__doc__ = func_doc + return fn + + return docstring_decorator + + +def copy_func(f): + """Returns a copy of a function f.""" + # Based on http://stackoverflow.com/a/6528148/190597 (Glenn Maynard) + g = types.FunctionType(f.__code__, f.__globals__, name=f.__name__, argdefs=f.__defaults__, closure=f.__closure__) + g = functools.update_wrapper(g, f) + g.__kwdefaults__ = f.__kwdefaults__ + return g diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dynamic_module_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dynamic_module_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9a72ab52e8fc7cad38ecaf6c719a325b3625093b --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/dynamic_module_utils.py @@ -0,0 +1,702 @@ +# Copyright 2021 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities to dynamically load objects from the Hub.""" + +import ast +import filecmp +import hashlib +import importlib +import importlib.util +import os +import re +import shutil +import signal +import sys +import threading +import warnings +from pathlib import Path +from types import ModuleType +from typing import Any, Optional, Union + +from huggingface_hub import try_to_load_from_cache + +from .utils import ( + HF_MODULES_CACHE, + TRANSFORMERS_DYNAMIC_MODULE_NAME, + cached_file, + extract_commit_hash, + is_offline_mode, + logging, +) + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name +_HF_REMOTE_CODE_LOCK = threading.Lock() + + +def init_hf_modules(): + """ + Creates the cache directory for modules with an init, and adds it to the Python path. + """ + # This function has already been executed if HF_MODULES_CACHE already is in the Python path. + if HF_MODULES_CACHE in sys.path: + return + + sys.path.append(HF_MODULES_CACHE) + os.makedirs(HF_MODULES_CACHE, exist_ok=True) + init_path = Path(HF_MODULES_CACHE) / "__init__.py" + if not init_path.exists(): + init_path.touch() + importlib.invalidate_caches() + + +def create_dynamic_module(name: Union[str, os.PathLike]) -> None: + """ + Creates a dynamic module in the cache directory for modules. + + Args: + name (`str` or `os.PathLike`): + The name of the dynamic module to create. + """ + init_hf_modules() + dynamic_module_path = (Path(HF_MODULES_CACHE) / name).resolve() + # If the parent module does not exist yet, recursively create it. + if not dynamic_module_path.parent.exists(): + create_dynamic_module(dynamic_module_path.parent) + os.makedirs(dynamic_module_path, exist_ok=True) + init_path = dynamic_module_path / "__init__.py" + if not init_path.exists(): + init_path.touch() + # It is extremely important to invalidate the cache when we change stuff in those modules, or users end up + # with errors about module that do not exist. Same for all other `invalidate_caches` in this file. + importlib.invalidate_caches() + + +def get_relative_imports(module_file: Union[str, os.PathLike]) -> list[str]: + """ + Get the list of modules that are relatively imported in a module file. + + Args: + module_file (`str` or `os.PathLike`): The module file to inspect. + + Returns: + `list[str]`: The list of relative imports in the module. + """ + with open(module_file, encoding="utf-8") as f: + content = f.read() + + # Imports of the form `import .xxx` + relative_imports = re.findall(r"^\s*import\s+\.(\S+)\s*$", content, flags=re.MULTILINE) + # Imports of the form `from .xxx import yyy` + relative_imports += re.findall(r"^\s*from\s+\.(\S+)\s+import", content, flags=re.MULTILINE) + # Unique-ify + return list(set(relative_imports)) + + +def get_relative_import_files(module_file: Union[str, os.PathLike]) -> list[str]: + """ + Get the list of all files that are needed for a given module. Note that this function recurses through the relative + imports (if a imports b and b imports c, it will return module files for b and c). + + Args: + module_file (`str` or `os.PathLike`): The module file to inspect. + + Returns: + `list[str]`: The list of all relative imports a given module needs (recursively), which will give us the list + of module files a given module needs. + """ + no_change = False + files_to_check = [module_file] + all_relative_imports = [] + + # Let's recurse through all relative imports + while not no_change: + new_imports = [] + for f in files_to_check: + new_imports.extend(get_relative_imports(f)) + + module_path = Path(module_file).parent + new_import_files = [str(module_path / m) for m in new_imports] + new_import_files = [f for f in new_import_files if f not in all_relative_imports] + files_to_check = [f"{f}.py" for f in new_import_files] + + no_change = len(new_import_files) == 0 + all_relative_imports.extend(files_to_check) + + return all_relative_imports + + +def get_imports(filename: Union[str, os.PathLike]) -> list[str]: + """ + Extracts all the libraries (not relative imports this time) that are imported in a file. + + Args: + filename (`str` or `os.PathLike`): The module file to inspect. + + Returns: + `list[str]`: The list of all packages required to use the input module. + """ + with open(filename, encoding="utf-8") as f: + content = f.read() + imported_modules = set() + + def recursive_look_for_imports(node): + if isinstance(node, ast.Try): + return # Don't recurse into Try blocks and ignore imports in them + elif isinstance(node, ast.If): + test = node.test + for condition_node in ast.walk(test): + if isinstance(condition_node, ast.Call) and getattr(condition_node.func, "id", "").startswith( + "is_flash_attn" + ): + # Don't recurse into "if flash_attn_available()" blocks and ignore imports in them + return + elif isinstance(node, ast.Import): + # Handle 'import x' statements + for alias in node.names: + top_module = alias.name.split(".")[0] + if top_module: + imported_modules.add(top_module) + elif isinstance(node, ast.ImportFrom): + # Handle 'from x import y' statements, ignoring relative imports + if node.level == 0 and node.module: + top_module = node.module.split(".")[0] + if top_module: + imported_modules.add(top_module) + + # Recursively visit all children + for child in ast.iter_child_nodes(node): + recursive_look_for_imports(child) + + tree = ast.parse(content) + recursive_look_for_imports(tree) + + return sorted(imported_modules) + + +def check_imports(filename: Union[str, os.PathLike]) -> list[str]: + """ + Check if the current Python environment contains all the libraries that are imported in a file. Will raise if a + library is missing. + + Args: + filename (`str` or `os.PathLike`): The module file to check. + + Returns: + `list[str]`: The list of relative imports in the file. + """ + imports = get_imports(filename) + missing_packages = [] + for imp in imports: + try: + importlib.import_module(imp) + except ImportError as exception: + logger.warning(f"Encountered exception while importing {imp}: {exception}") + # Some packages can fail with an ImportError because of a dependency issue. + # This check avoids hiding such errors. + # See https://github.com/huggingface/transformers/issues/33604 + if "No module named" in str(exception): + missing_packages.append(imp) + else: + raise + + if len(missing_packages) > 0: + raise ImportError( + "This modeling file requires the following packages that were not found in your environment: " + f"{', '.join(missing_packages)}. Run `pip install {' '.join(missing_packages)}`" + ) + + return get_relative_imports(filename) + + +def get_class_in_module( + class_name: str, + module_path: Union[str, os.PathLike], + *, + force_reload: bool = False, +) -> type: + """ + Import a module on the cache directory for modules and extract a class from it. + + Args: + class_name (`str`): The name of the class to import. + module_path (`str` or `os.PathLike`): The path to the module to import. + force_reload (`bool`, *optional*, defaults to `False`): + Whether to reload the dynamic module from file if it already exists in `sys.modules`. + Otherwise, the module is only reloaded if the file has changed. + + Returns: + `typing.Type`: The class looked for. + """ + name = os.path.normpath(module_path) + if name.endswith(".py"): + name = name[:-3] + name = name.replace(os.path.sep, ".") + module_file: Path = Path(HF_MODULES_CACHE) / module_path + with _HF_REMOTE_CODE_LOCK: + if force_reload: + sys.modules.pop(name, None) + importlib.invalidate_caches() + cached_module: Optional[ModuleType] = sys.modules.get(name) + module_spec = importlib.util.spec_from_file_location(name, location=module_file) + + # Hash the module file and all its relative imports to check if we need to reload it + module_files: list[Path] = [module_file] + sorted(map(Path, get_relative_import_files(module_file))) + module_hash: str = hashlib.sha256(b"".join(bytes(f) + f.read_bytes() for f in module_files)).hexdigest() + + module: ModuleType + if cached_module is None: + module = importlib.util.module_from_spec(module_spec) + # insert it into sys.modules before any loading begins + sys.modules[name] = module + else: + module = cached_module + # reload in both cases, unless the module is already imported and the hash hits + if getattr(module, "__transformers_module_hash__", "") != module_hash: + module_spec.loader.exec_module(module) + module.__transformers_module_hash__ = module_hash + return getattr(module, class_name) + + +def get_cached_module_file( + pretrained_model_name_or_path: Union[str, os.PathLike], + module_file: str, + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + resume_download: Optional[bool] = None, + proxies: Optional[dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + local_files_only: bool = False, + repo_type: Optional[str] = None, + _commit_hash: Optional[str] = None, + **deprecated_kwargs, +) -> str: + """ + Prepares Downloads a module from a local folder or a distant repo and returns its path inside the cached + Transformers module. + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a configuration file saved using the + [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. + + module_file (`str`): + The name of the module file containing the class to look for. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + repo_type (`str`, *optional*): + Specify the repo type (useful when downloading from a space for instance). + + + + Passing `token=True` is required when you want to use a private model. + + + + Returns: + `str`: The path to the module inside the cache. + """ + use_auth_token = deprecated_kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + token = use_auth_token + + if is_offline_mode() and not local_files_only: + logger.info("Offline mode: forcing local_files_only=True") + local_files_only = True + + # Download and cache module_file from the repo `pretrained_model_name_or_path` of grab it if it's a local file. + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + is_local = os.path.isdir(pretrained_model_name_or_path) + if is_local: + submodule = os.path.basename(pretrained_model_name_or_path) + else: + submodule = pretrained_model_name_or_path.replace("/", os.path.sep) + cached_module = try_to_load_from_cache( + pretrained_model_name_or_path, module_file, cache_dir=cache_dir, revision=_commit_hash, repo_type=repo_type + ) + + new_files = [] + try: + # Load from URL or cache if already cached + resolved_module_file = cached_file( + pretrained_model_name_or_path, + module_file, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + revision=revision, + repo_type=repo_type, + _commit_hash=_commit_hash, + ) + if not is_local and cached_module != resolved_module_file: + new_files.append(module_file) + + except OSError: + logger.error(f"Could not locate the {module_file} inside {pretrained_model_name_or_path}.") + raise + + # Check we have all the requirements in our environment + modules_needed = check_imports(resolved_module_file) + + # Now we move the module inside our cached dynamic modules. + full_submodule = TRANSFORMERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule + create_dynamic_module(full_submodule) + submodule_path = Path(HF_MODULES_CACHE) / full_submodule + if submodule == os.path.basename(pretrained_model_name_or_path): + # We copy local files to avoid putting too many folders in sys.path. This copy is done when the file is new or + # has changed since last copy. + if not (submodule_path / module_file).exists() or not filecmp.cmp( + resolved_module_file, str(submodule_path / module_file) + ): + shutil.copy(resolved_module_file, submodule_path / module_file) + importlib.invalidate_caches() + for module_needed in modules_needed: + module_needed = f"{module_needed}.py" + module_needed_file = os.path.join(pretrained_model_name_or_path, module_needed) + if not (submodule_path / module_needed).exists() or not filecmp.cmp( + module_needed_file, str(submodule_path / module_needed) + ): + shutil.copy(module_needed_file, submodule_path / module_needed) + importlib.invalidate_caches() + else: + # Get the commit hash + commit_hash = extract_commit_hash(resolved_module_file, _commit_hash) + + # The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the + # benefit of versioning. + submodule_path = submodule_path / commit_hash + full_submodule = full_submodule + os.path.sep + commit_hash + create_dynamic_module(full_submodule) + + if not (submodule_path / module_file).exists(): + shutil.copy(resolved_module_file, submodule_path / module_file) + importlib.invalidate_caches() + # Make sure we also have every file with relative + for module_needed in modules_needed: + if not (submodule_path / f"{module_needed}.py").exists(): + get_cached_module_file( + pretrained_model_name_or_path, + f"{module_needed}.py", + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + _commit_hash=commit_hash, + ) + new_files.append(f"{module_needed}.py") + + if len(new_files) > 0 and revision is None: + new_files = "\n".join([f"- {f}" for f in new_files]) + repo_type_str = "" if repo_type is None else f"{repo_type}s/" + url = f"https://huggingface.co/{repo_type_str}{pretrained_model_name_or_path}" + logger.warning( + f"A new version of the following files was downloaded from {url}:\n{new_files}" + "\n. Make sure to double-check they do not contain any added malicious code. To avoid downloading new " + "versions of the code file, you can pin a revision." + ) + + return os.path.join(full_submodule, module_file) + + +def get_class_from_dynamic_module( + class_reference: str, + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + resume_download: Optional[bool] = None, + proxies: Optional[dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + local_files_only: bool = False, + repo_type: Optional[str] = None, + code_revision: Optional[str] = None, + **kwargs, +) -> type: + """ + Extracts a class from a module file, present in the local folder or repository of a model. + + + + Calling this function will execute the code in the module file found locally or downloaded from the Hub. It should + therefore only be called on trusted repos. + + + + + + Args: + class_reference (`str`): + The full name of the class to load, including its module and optionally its repo. + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a configuration file saved using the + [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. + + This is used when `class_reference` does not specify another repo. + module_file (`str`): + The name of the module file containing the class to look for. + class_name (`str`): + The name of the class to import in the module. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + repo_type (`str`, *optional*): + Specify the repo type (useful when downloading from a space for instance). + code_revision (`str`, *optional*, defaults to `"main"`): + The specific revision to use for the code on the Hub, if the code leaves in a different repository than the + rest of the model. It can be a branch name, a tag name, or a commit id, since we use a git-based system for + storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. + + + + Passing `token=True` is required when you want to use a private model. + + + + Returns: + `typing.Type`: The class, dynamically imported from the module. + + Examples: + + ```python + # Download module `modeling.py` from huggingface.co and cache then extract the class `MyBertModel` from this + # module. + cls = get_class_from_dynamic_module("modeling.MyBertModel", "sgugger/my-bert-model") + + # Download module `modeling.py` from a given repo and cache then extract the class `MyBertModel` from this + # module. + cls = get_class_from_dynamic_module("sgugger/my-bert-model--modeling.MyBertModel", "sgugger/another-bert-model") + ```""" + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + token = use_auth_token + + # Catch the name of the repo if it's specified in `class_reference` + if "--" in class_reference: + repo_id, class_reference = class_reference.split("--") + else: + repo_id = pretrained_model_name_or_path + module_file, class_name = class_reference.split(".") + + if code_revision is None and pretrained_model_name_or_path == repo_id: + code_revision = revision + # And lastly we get the class inside our newly created module + final_module = get_cached_module_file( + repo_id, + module_file + ".py", + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + revision=code_revision, + local_files_only=local_files_only, + repo_type=repo_type, + ) + return get_class_in_module(class_name, final_module, force_reload=force_download) + + +def custom_object_save(obj: Any, folder: Union[str, os.PathLike], config: Optional[dict] = None) -> list[str]: + """ + Save the modeling files corresponding to a custom model/configuration/tokenizer etc. in a given folder. Optionally + adds the proper fields in a config. + + Args: + obj (`Any`): The object for which to save the module files. + folder (`str` or `os.PathLike`): The folder where to save. + config (`PretrainedConfig` or dictionary, `optional`): + A config in which to register the auto_map corresponding to this custom object. + + Returns: + `List[str]`: The list of files saved. + """ + if obj.__module__ == "__main__": + logger.warning( + f"We can't save the code defining {obj} in {folder} as it's been defined in __main__. You should put " + "this code in a separate module so we can include it in the saved folder and make it easier to share via " + "the Hub." + ) + return + + def _set_auto_map_in_config(_config): + module_name = obj.__class__.__module__ + last_module = module_name.split(".")[-1] + full_name = f"{last_module}.{obj.__class__.__name__}" + # Special handling for tokenizers + if "Tokenizer" in full_name: + slow_tokenizer_class = None + fast_tokenizer_class = None + if obj.__class__.__name__.endswith("Fast"): + # Fast tokenizer: we have the fast tokenizer class and we may have the slow one has an attribute. + fast_tokenizer_class = f"{last_module}.{obj.__class__.__name__}" + if getattr(obj, "slow_tokenizer_class", None) is not None: + slow_tokenizer = getattr(obj, "slow_tokenizer_class") + slow_tok_module_name = slow_tokenizer.__module__ + last_slow_tok_module = slow_tok_module_name.split(".")[-1] + slow_tokenizer_class = f"{last_slow_tok_module}.{slow_tokenizer.__name__}" + else: + # Slow tokenizer: no way to have the fast class + slow_tokenizer_class = f"{last_module}.{obj.__class__.__name__}" + + full_name = (slow_tokenizer_class, fast_tokenizer_class) + + if isinstance(_config, dict): + auto_map = _config.get("auto_map", {}) + auto_map[obj._auto_class] = full_name + _config["auto_map"] = auto_map + elif getattr(_config, "auto_map", None) is not None: + _config.auto_map[obj._auto_class] = full_name + else: + _config.auto_map = {obj._auto_class: full_name} + + # Add object class to the config auto_map + if isinstance(config, (list, tuple)): + for cfg in config: + _set_auto_map_in_config(cfg) + elif config is not None: + _set_auto_map_in_config(config) + + result = [] + # Copy module file to the output folder. + object_file = sys.modules[obj.__module__].__file__ + dest_file = Path(folder) / (Path(object_file).name) + shutil.copy(object_file, dest_file) + result.append(dest_file) + + # Gather all relative imports recursively and make sure they are copied as well. + for needed_file in get_relative_import_files(object_file): + dest_file = Path(folder) / (Path(needed_file).name) + shutil.copy(needed_file, dest_file) + result.append(dest_file) + + return result + + +def _raise_timeout_error(signum, frame): + raise ValueError( + "Loading this model requires you to execute custom code contained in the model repository on your local " + "machine. Please set the option `trust_remote_code=True` to permit loading of this model." + ) + + +TIME_OUT_REMOTE_CODE = 15 + + +def resolve_trust_remote_code(trust_remote_code, model_name, has_local_code, has_remote_code): + if trust_remote_code is None: + if has_local_code: + trust_remote_code = False + elif has_remote_code and TIME_OUT_REMOTE_CODE > 0: + prev_sig_handler = None + try: + prev_sig_handler = signal.signal(signal.SIGALRM, _raise_timeout_error) + signal.alarm(TIME_OUT_REMOTE_CODE) + while trust_remote_code is None: + answer = input( + f"The repository for {model_name} contains custom code which must be executed to correctly " + f"load the model. You can inspect the repository content at https://hf.co/{model_name}.\n" + f"You can avoid this prompt in future by passing the argument `trust_remote_code=True`.\n\n" + f"Do you wish to run the custom code? [y/N] " + ) + if answer.lower() in ["yes", "y", "1"]: + trust_remote_code = True + elif answer.lower() in ["no", "n", "0", ""]: + trust_remote_code = False + signal.alarm(0) + except Exception: + # OS which does not support signal.SIGALRM + raise ValueError( + f"The repository for {model_name} contains custom code which must be executed to correctly " + f"load the model. You can inspect the repository content at https://hf.co/{model_name}.\n" + f"Please pass the argument `trust_remote_code=True` to allow custom code to be run." + ) + finally: + if prev_sig_handler is not None: + signal.signal(signal.SIGALRM, prev_sig_handler) + signal.alarm(0) + elif has_remote_code: + # For the CI which puts the timeout at 0 + _raise_timeout_error(None, None) + + if has_remote_code and not has_local_code and not trust_remote_code: + raise ValueError( + f"Loading {model_name} requires you to execute the configuration file in that" + " repo on your local machine. Make sure you have read the code there to avoid malicious use, then" + " set the option `trust_remote_code=True` to remove this error." + ) + + return trust_remote_code diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/eetq.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/eetq.py new file mode 100644 index 0000000000000000000000000000000000000000..97698cf1aa37c6279d4f6932ef496ed708e324a7 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/eetq.py @@ -0,0 +1,121 @@ +# coding=utf-8 +# Copyright 2024 NetEase, Inc. and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..utils import is_accelerate_available, is_eetq_available, logging + + +if is_eetq_available(): + import eetq + import torch.nn as nn + +if is_accelerate_available(): + from accelerate import init_empty_weights + +logger = logging.get_logger(__name__) + + +def _replace_with_eetq_linear( + model, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + has_been_replaced=False, + pre_quantized=False, +): + """ + Private method that wraps the recursion for module replacement. + + Returns the converted model and a boolean that indicates if the conversion has been successfull or not. + """ + if current_key_name is None: + current_key_name = [] + + for name, module in model.named_children(): + current_key_name.append(name) + + if (isinstance(module, nn.Linear)) and name not in modules_to_not_convert: + # Check if the current key is not in the `modules_to_not_convert` + current_key_name_str = ".".join(current_key_name) + if not any( + (key + "." in current_key_name_str) or (key == current_key_name_str) for key in modules_to_not_convert + ): + with init_empty_weights(): + in_features = module.in_features + out_features = module.out_features + model._modules[name] = eetq.EetqLinear( + in_features, out_features, module.bias is not None, module.weight.device + ) + if pre_quantized: + model._modules[name].register_scale(module.weight.device) + has_been_replaced = True + + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) + if len(list(module.children())) > 0: + _, has_been_replaced = _replace_with_eetq_linear( + module, + modules_to_not_convert, + current_key_name, + quantization_config, + has_been_replaced=has_been_replaced, + pre_quantized=pre_quantized, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def replace_with_eetq_linear( + model, modules_to_not_convert=None, current_key_name=None, quantization_config=None, pre_quantized=False +): + """ + A helper function to replace all `torch.nn.Linear` modules by `eetq.EetqLinear` modules from the `eetq` + library. This will enable running your models using high performance int8 weight-only gemm kerner from + FasterTransformer and TensorRT-LLM. Make sure `eetq` compiled with the correct CUDA + version of your hardware is installed before running this function. EETQ shall be installed via the source + 'https://github.com/NetEase-FuXi/EETQ' + + The function will be run recursively and replace all `torch.nn.Linear` modules except for the `lm_head` that should + be kept as a `torch.nn.Linear` module. The replacement is done under `init_empty_weights` context manager so no + CPU/GPU memory is required to run this function. Each weight will be quantized along the channel. + + Parameters: + model (`torch.nn.Module`): + Input model or `torch.nn.Module` as the function is run recursively. + modules_to_not_convert (`List[`str`]`, *optional*, defaults to `["lm_head"]`): + Names of the modules to not convert in `EetqLinear`. In practice we keep the `lm_head` in full precision + for numerical stability reasons. + current_key_name (`List[`str`]`, *optional*): + An array to track the current key of the recursion. This is used to check whether the current key (part of + it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or + `disk`). + """ + + modules_to_not_convert = ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert + + if quantization_config.modules_to_not_convert is not None: + modules_to_not_convert.extend(quantization_config.modules_to_not_convert) + modules_to_not_convert = list(set(modules_to_not_convert)) + model, has_been_replaced = _replace_with_eetq_linear( + model, modules_to_not_convert, current_key_name, quantization_config, pre_quantized=pre_quantized + ) + + if not has_been_replaced: + logger.warning( + "You are loading your model using eetq but no linear modules were found in your model." + " Please double check your model architecture, or submit an issue on github if you think this is" + " a bug." + ) + + return model diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/eval_results.json b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..c10ca729e0cfa648e2a49c5af3e92785d447058e --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/eval_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 0.9633128262210255, + "eval_loss": 3.0188777446746826, + "eval_perplexity": 20.46830812210243, + "eval_runtime": 18.5237, + "eval_samples_per_second": 228.788, + "eval_steps_per_second": 7.18 +} \ No newline at end of file diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/executorch.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/executorch.py new file mode 100644 index 0000000000000000000000000000000000000000..591c556e59f07e39e758f04e29c74871930034d9 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/executorch.py @@ -0,0 +1,408 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from typing import Optional + +import torch + +from transformers.generation.configuration_utils import GenerationConfig + +from ..utils.import_utils import is_torch_available + + +if is_torch_available(): + from transformers import PreTrainedModel, StaticCache + from transformers.pytorch_utils import is_torch_greater_or_equal, is_torch_greater_or_equal_than_2_3 + + +class TorchExportableModuleWithStaticCache(torch.nn.Module): + """ + A wrapper module designed to make a `PreTrainedModel` exportable with `torch.export`, + specifically for use with static caching. This module ensures that the exported model + is compatible with further lowering and execution in `ExecuTorch`. + + Note: + This class is specifically designed to support export process using `torch.export` + in a way that ensures the model can be further lowered and run efficiently in `ExecuTorch`. + """ + + def __init__(self, model: PreTrainedModel): + """ + Initializes the wrapper module with the pretrained model. + + Args: + model (`PreTrainedModel`): The pretrained model to wrap. The model must have caching + enabled and use a 'static' caching implementation. + + Raises: + AssertionError: If the pretrained model does not have caching enabled or if it does + not use a 'static' caching implementation in `model.generation_config`. + """ + super().__init__() + + # Sanity checks + if model.generation_config is None: + raise AssertionError( + "The model must have a generation config to be exported with static caching. " + "Please set `generation_config`." + ) + + if not model.generation_config.use_cache: + raise AssertionError( + "The model must have caching enabled to be exported with static caching. " + "Please set `generation_config.use_cache=True`." + ) + + if model.generation_config.cache_implementation != "static": + raise AssertionError( + "The model must use a 'static' caching implementation to be exported with static caching. " + "Please set `generation_config.cache_implementation='static'`." + ) + + self.model = model + self.static_cache = StaticCache( + config=self.model.config, + max_batch_size=self.model.generation_config.cache_config.batch_size, + max_cache_len=self.model.generation_config.cache_config.max_cache_len, + device=self.model.generation_config.cache_config.device, + dtype=self.model.dtype, + ) + for i in range(len(self.static_cache.key_cache)): + self.register_buffer(f"key_cache_{i}", self.static_cache.key_cache[i], persistent=False) + self.register_buffer(f"value_cache_{i}", self.static_cache.value_cache[i], persistent=False) + + self.is_causal = any("CausalLM" in arch for arch in self.model.config.architectures) + if self.is_causal: + causal_mask = torch.tril( + torch.ones( + self.static_cache.max_cache_len, + self.static_cache.max_cache_len, + dtype=torch.bool, + ) + ) + self.register_buffer("mask", causal_mask, persistent=False) + + def forward(self, input_ids: torch.Tensor, cache_position: torch.Tensor): + """ + Forward pass of the module, which is compatible with the ExecuTorch runtime. + + Args: + input_ids (`torch.Tensor`): Tensor representing current input token id to the module. + cache_position (`torch.Tensor`): Tensor representing current input position in the cache. + + Returns: + torch.Tensor: Logits output from the model. + + This forward adapter serves two primary purposes: + + 1. **Making the Model `torch.export`-Compatible**: + The adapter hides unsupported objects, such as the `Cache`, from the graph inputs and outputs, + enabling the model to be exportable using `torch.export` without encountering issues. + + 2. **Ensuring Compatibility with `ExecuTorch` runtime**: + The adapter matches the model's forward signature with that in `executorch/extension/llm/runner`, + ensuring that the exported model can be executed in `ExecuTorch` out-of-the-box. + """ + _, seqlen = input_ids.shape + attn_mask = self.mask[cache_position, :seqlen] if self.is_causal else None + position_ids = cache_position.unsqueeze(0) + past_key_values = self.static_cache + + outs = self.model( + input_ids=input_ids, + attention_mask=attn_mask, + position_ids=position_ids, + cache_position=cache_position, + past_key_values=past_key_values, + use_cache=True, + ) + return outs.logits + + @staticmethod + def generate( + exported_program: torch.export.ExportedProgram, prompt_token_ids: torch.Tensor, max_new_tokens: int + ) -> torch.Tensor: + """ + Generate a sequence of tokens using an exported program. + + This util function is designed to test exported models by simulating the generation process. + It processes the input prompt tokens sequentially (no parallel prefill). + This generate function is not intended to replace the original `generate` method, and the support + for leveraging the original `generate` is potentially planed! + + Args: + exported_program (`torch.export.ExportedProgram`): The exported program generated via `torch.export`. + prompt_token_ids (`torch.Tensor`): Tensor representing the input prompt token IDs. + max_new_tokens (`int`): Maximum number of new tokens to generate. Note that the total generation + length is limited by both `max_new_tokens` and the model's cache size. + + Returns: + torch.Tensor: A tensor containing the generated sequence of token IDs, including the original prompt tokens. + """ + prompt_token_len = prompt_token_ids.shape[-1] + max_generation_length = prompt_token_len + max_new_tokens + for buffer_name, buffer in exported_program.named_buffers(): + if buffer_name.startswith("key_cache"): + max_cache_len = buffer.shape[2] + max_generation_length = min(max_generation_length, max_cache_len) + break + + response_tokens = [] + for input_pos in range(min(max_generation_length, prompt_token_len)): + result = exported_program.module().forward( + input_ids=prompt_token_ids[:, input_pos : input_pos + 1], + cache_position=torch.tensor([input_pos], dtype=torch.long), + ) + response_tokens.append(prompt_token_ids[0][input_pos].item()) + + current_token = torch.argmax(result[:, -1, :], dim=-1).item() + response_tokens.append(current_token) + + while len(response_tokens) < max_generation_length: + result = exported_program.module().forward( + input_ids=torch.tensor([[current_token]], dtype=torch.long), + cache_position=torch.tensor([len(response_tokens)], dtype=torch.long), + ) + current_token = torch.argmax(result[:, -1, :], dim=-1).item() + response_tokens.append(current_token) + + return torch.tensor([response_tokens], dtype=torch.long) + + +def convert_and_export_with_cache( + model: PreTrainedModel, + example_input_ids: Optional[torch.Tensor] = None, + example_cache_position: Optional[torch.Tensor] = None, +): + """ + Convert a `PreTrainedModel` into an exportable module and export it using `torch.export`, + ensuring the exported model is compatible with `ExecuTorch`. + + Args: + model (`PreTrainedModel`): The pretrained model to be exported. + example_input_ids (`torch.Tensor`): Example input token id used by `torch.export`. + example_cache_position (`torch.Tensor`): Example current cache position used by `torch.export`. + + Returns: + Exported program (`torch.export.ExportedProgram`): The exported program generated via `torch.export`. + """ + if not is_torch_greater_or_equal_than_2_3: + raise ImportError("torch >= 2.3 is required.") + + import torch.export._trace + + with torch.no_grad(): + # TODO: The default inputs only work for text models. We need to add support for vision/audio models. + example_input_ids = ( + example_input_ids if example_input_ids is not None else torch.tensor([[1]], dtype=torch.long) + ) + example_cache_position = ( + example_cache_position if example_cache_position is not None else torch.tensor([0], dtype=torch.long) + ) + + if is_torch_greater_or_equal("2.5.0"): + exported_program = torch.export.export( + TorchExportableModuleWithStaticCache(model), + args=(example_input_ids,), + kwargs={"cache_position": example_cache_position}, + strict=True, + ) + else: + # We have to keep this path for BC. + # + # Due to issue https://github.com/pytorch/pytorch/issues/128394, we need to switch to use an internal + # export API and pre_dispatch=False. Switch to use the public API once the issue is included in 2.5 release. + exported_program = torch.export._trace._export( + TorchExportableModuleWithStaticCache(model), + args=(example_input_ids,), + kwargs={"cache_position": example_cache_position}, + pre_dispatch=False, + strict=True, + ) + return exported_program + + +class Seq2SeqLMEncoderExportableModule(torch.nn.Module): + """ + A wrapper module designed to make a Seq2Seq LM encoder exportable with `torch.export`. + This module ensures that the exported encoder model is compatible with ExecuTorch. + """ + + def __init__(self, encoder_model): + super().__init__() + self.encoder = encoder_model + + def forward(self, input_ids): + return self.encoder(input_ids=input_ids).last_hidden_state + + +class Seq2SeqLMDecoderExportableModuleWithStaticCache(torch.nn.Module): + """ + A wrapper module designed to make a Seq2Seq LM decoder exportable with `torch.export`, + specifically for use with static caching. This module ensures the exported decoder + is compatible with ExecuTorch. + """ + + def __init__(self, model, max_static_cache_length, batch_size): + super().__init__() + + # Get the decoder component + self.decoder = model.get_decoder() + self.lm_head = model.lm_head + self.config = model.config + + # Initialize static cache + self.static_cache = StaticCache( + config=self.config, + max_batch_size=batch_size, + max_cache_len=max_static_cache_length, + device="cpu", + dtype=torch.float32, + ) + + # Register cache buffers to make them exportable + for i in range(len(self.static_cache.key_cache)): + self.register_buffer(f"key_cache_{i}", self.static_cache.key_cache[i], persistent=False) + self.register_buffer(f"value_cache_{i}", self.static_cache.value_cache[i], persistent=False) + + def forward(self, decoder_input_ids, encoder_hidden_states, cache_position): + # Get outputs from decoder + outputs = self.decoder( + input_ids=decoder_input_ids, + encoder_hidden_states=encoder_hidden_states, + past_key_values=self.static_cache, + use_cache=True, + cache_position=cache_position, + ) + + # Apply language model head + lm_logits = self.lm_head(outputs[0]) + + return lm_logits + + +class Seq2SeqLMExportableModule(torch.nn.Module): + def __init__( + self, model, batch_size=1, max_hidden_seq_length=4096, cache_implementation="static", max_cache_length=1024 + ): + super().__init__() + + self.full_model = model + self.encoder = model.get_encoder() + self.config = model.config + self.max_hidden_seq_length = max_hidden_seq_length + self.generation_config = GenerationConfig( + use_cache=True, + max_length=max_cache_length, + cache_implementation=cache_implementation, + cache_config={ + "batch_size": batch_size, + "max_cache_len": max_cache_length, + }, + ) + self.exported_encoder = None + self.exported_decoder = None + + def _export_encoder(self, encoder_input_ids): + wrapped_encoder = Seq2SeqLMEncoderExportableModule(self.encoder).to("cpu").eval() + + # Define dynamic sequence length for encoder + seq_len_dim = torch.export.Dim("encoder_seq_length", max=self.max_hidden_seq_length) + + # Export the encoder + with torch.no_grad(): + exported_encoder = torch.export.export( + wrapped_encoder, (encoder_input_ids,), dynamic_shapes={"input_ids": {1: seq_len_dim}}, strict=True + ) + + return exported_encoder + + def _export_decoder(self, decoder_input_ids, encoder_hidden_states, cache_position): + wrapped_decoder = ( + Seq2SeqLMDecoderExportableModuleWithStaticCache( + model=self.full_model, + max_static_cache_length=self.generation_config.cache_config.max_cache_len, + batch_size=self.generation_config.cache_config.batch_size, + ) + .to("cpu") + .eval() + ) + + # Define dynamic dimension for encoder output sequence length + encoder_seq_len_dim = torch.export.Dim("encoder_hidden_seq_length", max=self.max_hidden_seq_length) + + # Export the decoder + with torch.no_grad(): + exported_decoder = torch.export.export( + wrapped_decoder, + (decoder_input_ids, encoder_hidden_states, cache_position), + dynamic_shapes={ + "decoder_input_ids": None, + "encoder_hidden_states": {1: encoder_seq_len_dim}, + "cache_position": None, + }, + strict=True, + ) + + return exported_decoder + + def export(self, encoder_input_ids=None, decoder_input_ids=None, encoder_hidden_states=None, cache_position=None): + example_encoder_input_ids = ( + encoder_input_ids if encoder_input_ids is not None else torch.ones((1, 10), dtype=torch.long) + ) + example_decoder_input_ids = ( + decoder_input_ids if decoder_input_ids is not None else torch.tensor([[0]], dtype=torch.long) + ) # Start token + example_cache_position = cache_position if cache_position is not None else torch.tensor([0], dtype=torch.long) + example_encoder_hidden_states = ( + encoder_hidden_states + if encoder_hidden_states is not None + else torch.zeros( + (self.generation_config.cache_config.batch_size, 10, self.config.d_model), dtype=torch.float32 + ) + ) + self.exported_encoder = self._export_encoder(example_encoder_input_ids) + self.exported_decoder = self._export_decoder( + example_decoder_input_ids, example_encoder_hidden_states, example_cache_position + ) + + # Return self to allow chaining + return self + + def generate(self, prompt_token_ids, max_new_tokens): + with torch.no_grad(): + # Run encoder + encoder_output = self.exported_encoder.module()(prompt_token_ids) + + # Initialize with start token (0 for T5) + decoder_input_ids = torch.tensor([[0]], dtype=torch.long) + generated_ids = [0] + + # Generate tokens one by one + for i in range(max_new_tokens - 1): + # Run decoder for next token prediction + logits = self.exported_decoder.module()( + decoder_input_ids, encoder_output, torch.tensor([i], dtype=torch.long) + ) + + # Get next token + next_token = torch.argmax(logits[:, -1, :], dim=-1).item() + generated_ids.append(next_token) + + # Update input for next iteration + decoder_input_ids = torch.tensor([[next_token]], dtype=torch.long) + + # Check if EOS token + if next_token == self.config.eos_token_id: + break + + return generated_ids diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/fbgemm_fp8.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/fbgemm_fp8.py new file mode 100644 index 0000000000000000000000000000000000000000..1cc5a8b23aa53127e2be855fef6e753ed5c5415f --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/fbgemm_fp8.py @@ -0,0 +1,284 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..activations import ACT2FN +from ..utils import is_accelerate_available, is_fbgemm_gpu_available, is_torch_available, logging + + +if is_torch_available(): + import torch + from torch import nn + +if is_accelerate_available(): + from accelerate import init_empty_weights + +if is_fbgemm_gpu_available(): + import fbgemm_gpu.experimental.gen_ai # noqa: F401 + +logger = logging.get_logger(__name__) + + +class FbgemmFp8Linear(torch.nn.Linear): + def __init__(self, in_features, out_features, bias, weight_dtype=torch.float32): + super().__init__(in_features, out_features, bias) + self.in_features = in_features + self.out_features = out_features + + self.weight = torch.nn.Parameter(torch.zeros((out_features, in_features), dtype=torch.float8_e4m3fn)) + self.weight_scale = torch.nn.Parameter(torch.zeros((out_features, 1), dtype=weight_dtype)) + self.register_buffer("input_scale_ub", torch.zeros([1], dtype=torch.float), persistent=False) + + if bias: + self.bias = torch.nn.Parameter(torch.zeros((self.out_features), dtype=weight_dtype)) + else: + self.bias = None + + def forward(self, x): + # quantize_fp8_per_row will squash the leading dimensions, so save the desired shape here + output_shape = (*x.shape[:-1], -1) + # x_quantized and x_scale are not necessarily on the same device as x, this is an issue. + # https://github.com/pytorch/FBGEMM/blob/e08af8539c391437f447173863df0f3f6f6f1855/fbgemm_gpu/experimental/gen_ai/src/quantize/quantize.cu#L1237C3-L1237C45 + x_quantized, x_scale = torch.ops.fbgemm.quantize_fp8_per_row( + x.view(-1, x.shape[-1]).contiguous(), scale_ub=self.input_scale_ub + ) + # moving x_quantized, x_scale here creates glibberish output ... However, if we move the output, it works + # x_quantized, x_scale = x_quantized.to(x.device), x_scale.to(x.device) + + # The computation still happens on the device where self.weight is even if x_quantized is not on the same device as self.weight + weight_scale_float32 = self.weight_scale.to(torch.float32) + output = torch.ops.fbgemm.f8f8bf16_rowwise( + x_quantized, self.weight, x_scale, weight_scale_float32, use_fast_accum=True + ) + output = output + self.bias if self.bias is not None else output + # Hacky for now, we have the output to the device of x + output = output.to(x.device) + output = output.reshape(output_shape) + del x_quantized, x_scale + return output + + +class FbgemmFp8Llama4TextExperts(nn.Module): + def __init__(self, config, dtype=torch.float32): + super().__init__() + self.num_experts = config.num_local_experts + self.intermediate_size = config.intermediate_size + self.hidden_size = config.hidden_size + self.expert_dim = self.intermediate_size + self.act_fn = ACT2FN[config.hidden_act] + # Register FP8 buffers for gate_up_proj + self.gate_up_proj = torch.nn.Parameter( + torch.zeros((self.num_experts, self.hidden_size, 2 * self.expert_dim), dtype=torch.float8_e4m3fn) + ) + self.gate_up_proj_scale = torch.nn.Parameter( + torch.zeros((self.num_experts, 1, self.expert_dim * 2), dtype=torch.float32) + ) + # Register FP8 buffers for down_proj + self.down_proj = torch.nn.Parameter( + torch.zeros((self.num_experts, self.expert_dim, self.hidden_size), dtype=torch.float8_e4m3fn) + ) + self.down_proj_scale = torch.nn.Parameter( + torch.zeros((self.num_experts, self.hidden_size, 1), dtype=torch.float32) + ) + # Register input scale upper bound + self.register_buffer("input_scale_ub", torch.zeros([1], dtype=torch.float), persistent=False) + + def forward(self, hidden_states): + """ + Args: + hidden_states (torch.Tensor): (batch_size * token_num, hidden_size) + Returns: + torch.Tensor: (batch_size * token_num, hidden_size) + """ + # Reshape hidden states for expert computation + hidden_states = hidden_states.view(self.num_experts, -1, self.hidden_size) + num_tokens = None + + # Pre-allocate tensor for all expert outputs with same shape as hidden_states + next_states = torch.empty_like(hidden_states) + + for i in range(self.num_experts): + # Extract expert's hidden states + expert_hidden = hidden_states[i] + expert_hidden_reshaped = expert_hidden.reshape(-1, self.hidden_size) + # Quantize for this expert + expert_quantized, expert_scale = torch.ops.fbgemm.quantize_fp8_per_row( + expert_hidden_reshaped, num_tokens, self.input_scale_ub + ) + sharded_expert_dim = self.gate_up_proj.shape[-1] // 2 + gate_up_proj_scale_float32 = self.gate_up_proj_scale.to(torch.float32) + + gate = torch.ops.fbgemm.f8f8bf16_rowwise( + expert_quantized, + self.gate_up_proj[i].transpose(0, 1)[:sharded_expert_dim].contiguous(), + expert_scale, + gate_up_proj_scale_float32[i][0][:sharded_expert_dim].view(-1, 1).contiguous(), + use_fast_accum=True, + ) + + up = torch.ops.fbgemm.f8f8bf16_rowwise( + expert_quantized, + self.gate_up_proj[i].transpose(0, 1)[sharded_expert_dim:].contiguous(), + expert_scale, + gate_up_proj_scale_float32[i][0][sharded_expert_dim:].view(-1, 1).contiguous(), + use_fast_accum=True, + ) + + activated = up * self.act_fn(gate) + + activated_quantized, activated_scale = torch.ops.fbgemm.quantize_fp8_per_row( + activated, num_tokens, self.input_scale_ub + ) + + down_proj_scale_float32 = self.down_proj_scale.to(torch.float32) + expert_output = torch.ops.fbgemm.f8f8bf16_rowwise( + activated_quantized, + self.down_proj[i].transpose(0, 1).contiguous(), + activated_scale, + down_proj_scale_float32[i].view(-1, 1).contiguous(), + use_fast_accum=True, + ) + + next_states[i] = expert_output + next_states = next_states.to(hidden_states.device) + return next_states.view(-1, self.hidden_size) + + +def _replace_with_fbgemm_fp8_linear( + model, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + has_been_replaced=False, + pre_quantized=False, + config=None, + tp_plan=None, +): + """ + Private method that wraps the recursion for module replacement. + + Returns the converted model and a boolean that indicates if the conversion has been successfull or not. + """ + + import re + + if current_key_name is None: + current_key_name = [] + + for name, module in model.named_children(): + current_key_name.append(name) + + if (isinstance(module, nn.Linear)) and name not in modules_to_not_convert: + # Check if the current key is not in the `modules_to_not_convert` + current_key_name_str = ".".join(current_key_name) + if not any( + (key + "." in current_key_name_str) or (key == current_key_name_str) for key in modules_to_not_convert + ): + with init_empty_weights(include_buffers=True): + in_features = module.in_features + out_features = module.out_features + model._modules[name] = FbgemmFp8Linear( + in_features, + out_features, + module.bias is not None, + ) + has_been_replaced = True + + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) + # set non persistant buffer outside of init_empty_weights + model._modules[name].input_scale_ub = torch.tensor( + [quantization_config.activation_scale_ub], + dtype=torch.float, + ) + if module.__class__.__name__ == "Llama4TextExperts" and name not in modules_to_not_convert: + current_key_name_str = ".".join(current_key_name) + if not any( + (key + "." in current_key_name_str) or (key == current_key_name_str) for key in modules_to_not_convert + ): + with init_empty_weights(include_buffers=True): + tp_plan[re.sub(r"\d+", "*", current_key_name_str + ".down_proj_scale")] = None + model._modules[name] = FbgemmFp8Llama4TextExperts( + config.text_config, + ) + model._modules[name].input_scale_ub = torch.tensor( + [quantization_config.activation_scale_ub], dtype=torch.float + ) + + if len(list(module.children())) > 0: + _, has_been_replaced = _replace_with_fbgemm_fp8_linear( + module, + modules_to_not_convert, + current_key_name, + quantization_config, + has_been_replaced=has_been_replaced, + pre_quantized=pre_quantized, + config=config, + tp_plan=tp_plan, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def replace_with_fbgemm_fp8_linear( + model, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + pre_quantized=False, + config=None, + tp_plan=None, +): + """ + A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules. + This will enable running your models using high performance fp8 kernel from FBGEMM library. + + The function will be run recursively and replace all `torch.nn.Linear` modules except for the `lm_head` that should + be kept as a `torch.nn.Linear` module. The replacement is done under `init_empty_weights` context manager so no + CPU/GPU memory is required to run this function. Each weight will be quantized along the channel. + + Parameters: + model (`torch.nn.Module`): + Input model or `torch.nn.Module` as the function is run recursively. + modules_to_not_convert (`List[`str`]`, *optional*, defaults to `["lm_head"]`): + Names of the modules to not convert in `FP8Linear`. In practice we keep the `lm_head` in full precision + for numerical stability reasons. + current_key_name (`List[`str`]`, *optional*): + An array to track the current key of the recursion. This is used to check whether the current key (part of + it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or + `disk`). + """ + + modules_to_not_convert = ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert + + if quantization_config.modules_to_not_convert is not None: + modules_to_not_convert.extend(quantization_config.modules_to_not_convert) + modules_to_not_convert = list(set(modules_to_not_convert)) + model, has_been_replaced = _replace_with_fbgemm_fp8_linear( + model, + modules_to_not_convert, + current_key_name, + quantization_config, + pre_quantized=pre_quantized, + config=config, + tp_plan=tp_plan, + ) + if not has_been_replaced: + logger.warning( + "You are loading your model using FP8 quantization but no linear modules were found in your model." + " Please double check your model architecture, or submit an issue on github if you think this is" + " a bug." + ) + + return model diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_auto.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_auto.py new file mode 100644 index 0000000000000000000000000000000000000000..0b8b38bc347cb104b1593906debaa9fe2970bdf3 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_auto.py @@ -0,0 +1,408 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AutoFeatureExtractor class.""" + +import importlib +import json +import os +import warnings +from collections import OrderedDict +from typing import Dict, Optional, Union + +# Build the list of all feature extractors +from ...configuration_utils import PretrainedConfig +from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code +from ...feature_extraction_utils import FeatureExtractionMixin +from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, cached_file, logging +from .auto_factory import _LazyAutoMapping +from .configuration_auto import ( + CONFIG_MAPPING_NAMES, + AutoConfig, + model_type_to_module_name, + replace_list_option_in_docstrings, +) + + +logger = logging.get_logger(__name__) + +FEATURE_EXTRACTOR_MAPPING_NAMES = OrderedDict( + [ + ("audio-spectrogram-transformer", "ASTFeatureExtractor"), + ("beit", "BeitFeatureExtractor"), + ("chinese_clip", "ChineseCLIPFeatureExtractor"), + ("clap", "ClapFeatureExtractor"), + ("clip", "CLIPFeatureExtractor"), + ("clipseg", "ViTFeatureExtractor"), + ("clvp", "ClvpFeatureExtractor"), + ("conditional_detr", "ConditionalDetrFeatureExtractor"), + ("convnext", "ConvNextFeatureExtractor"), + ("cvt", "ConvNextFeatureExtractor"), + ("dac", "DacFeatureExtractor"), + ("data2vec-audio", "Wav2Vec2FeatureExtractor"), + ("data2vec-vision", "BeitFeatureExtractor"), + ("deformable_detr", "DeformableDetrFeatureExtractor"), + ("deit", "DeiTFeatureExtractor"), + ("detr", "DetrFeatureExtractor"), + ("dinat", "ViTFeatureExtractor"), + ("donut-swin", "DonutFeatureExtractor"), + ("dpt", "DPTFeatureExtractor"), + ("encodec", "EncodecFeatureExtractor"), + ("flava", "FlavaFeatureExtractor"), + ("glpn", "GLPNFeatureExtractor"), + ("groupvit", "CLIPFeatureExtractor"), + ("hubert", "Wav2Vec2FeatureExtractor"), + ("imagegpt", "ImageGPTFeatureExtractor"), + ("layoutlmv2", "LayoutLMv2FeatureExtractor"), + ("layoutlmv3", "LayoutLMv3FeatureExtractor"), + ("levit", "LevitFeatureExtractor"), + ("maskformer", "MaskFormerFeatureExtractor"), + ("mctct", "MCTCTFeatureExtractor"), + ("mimi", "EncodecFeatureExtractor"), + ("mobilenet_v1", "MobileNetV1FeatureExtractor"), + ("mobilenet_v2", "MobileNetV2FeatureExtractor"), + ("mobilevit", "MobileViTFeatureExtractor"), + ("moonshine", "Wav2Vec2FeatureExtractor"), + ("moshi", "EncodecFeatureExtractor"), + ("nat", "ViTFeatureExtractor"), + ("owlvit", "OwlViTFeatureExtractor"), + ("perceiver", "PerceiverFeatureExtractor"), + ("phi4_multimodal", "Phi4MultimodalFeatureExtractor"), + ("poolformer", "PoolFormerFeatureExtractor"), + ("pop2piano", "Pop2PianoFeatureExtractor"), + ("regnet", "ConvNextFeatureExtractor"), + ("resnet", "ConvNextFeatureExtractor"), + ("seamless_m4t", "SeamlessM4TFeatureExtractor"), + ("seamless_m4t_v2", "SeamlessM4TFeatureExtractor"), + ("segformer", "SegformerFeatureExtractor"), + ("sew", "Wav2Vec2FeatureExtractor"), + ("sew-d", "Wav2Vec2FeatureExtractor"), + ("speech_to_text", "Speech2TextFeatureExtractor"), + ("speecht5", "SpeechT5FeatureExtractor"), + ("swiftformer", "ViTFeatureExtractor"), + ("swin", "ViTFeatureExtractor"), + ("swinv2", "ViTFeatureExtractor"), + ("table-transformer", "DetrFeatureExtractor"), + ("timesformer", "VideoMAEFeatureExtractor"), + ("tvlt", "TvltFeatureExtractor"), + ("unispeech", "Wav2Vec2FeatureExtractor"), + ("unispeech-sat", "Wav2Vec2FeatureExtractor"), + ("univnet", "UnivNetFeatureExtractor"), + ("van", "ConvNextFeatureExtractor"), + ("videomae", "VideoMAEFeatureExtractor"), + ("vilt", "ViltFeatureExtractor"), + ("vit", "ViTFeatureExtractor"), + ("vit_mae", "ViTFeatureExtractor"), + ("vit_msn", "ViTFeatureExtractor"), + ("wav2vec2", "Wav2Vec2FeatureExtractor"), + ("wav2vec2-bert", "Wav2Vec2FeatureExtractor"), + ("wav2vec2-conformer", "Wav2Vec2FeatureExtractor"), + ("wavlm", "Wav2Vec2FeatureExtractor"), + ("whisper", "WhisperFeatureExtractor"), + ("xclip", "CLIPFeatureExtractor"), + ("yolos", "YolosFeatureExtractor"), + ] +) + +FEATURE_EXTRACTOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES) + + +def feature_extractor_class_from_name(class_name: str): + for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items(): + if class_name in extractors: + module_name = model_type_to_module_name(module_name) + + module = importlib.import_module(f".{module_name}", "transformers.models") + try: + return getattr(module, class_name) + except AttributeError: + continue + + for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items(): + if getattr(extractor, "__name__", None) == class_name: + return extractor + + # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main + # init and we return the proper dummy to get an appropriate error message. + main_module = importlib.import_module("transformers") + if hasattr(main_module, class_name): + return getattr(main_module, class_name) + + return None + + +def get_feature_extractor_config( + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + resume_download: Optional[bool] = None, + proxies: Optional[Dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + local_files_only: bool = False, + **kwargs, +): + """ + Loads the tokenizer configuration from a pretrained model tokenizer configuration. + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a configuration file saved using the + [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. + + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + + + + Passing `token=True` is required when you want to use a private model. + + + + Returns: + `Dict`: The configuration of the tokenizer. + + Examples: + + ```python + # Download configuration from huggingface.co and cache. + tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased") + # This model does not have a tokenizer config so the result will be an empty dict. + tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base") + + # Save a pretrained tokenizer locally and you can reload its config + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") + tokenizer.save_pretrained("tokenizer-test") + tokenizer_config = get_tokenizer_config("tokenizer-test") + ```""" + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + token = use_auth_token + + resolved_config_file = cached_file( + pretrained_model_name_or_path, + FEATURE_EXTRACTOR_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + _raise_exceptions_for_gated_repo=False, + _raise_exceptions_for_missing_entries=False, + _raise_exceptions_for_connection_errors=False, + ) + if resolved_config_file is None: + logger.info( + "Could not locate the feature extractor configuration file, will try to use the model config instead." + ) + return {} + + with open(resolved_config_file, encoding="utf-8") as reader: + return json.load(reader) + + +class AutoFeatureExtractor: + r""" + This is a generic feature extractor class that will be instantiated as one of the feature extractor classes of the + library when created with the [`AutoFeatureExtractor.from_pretrained`] class method. + + This class cannot be instantiated directly using `__init__()` (throws an error). + """ + + def __init__(self): + raise EnvironmentError( + "AutoFeatureExtractor is designed to be instantiated " + "using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method." + ) + + @classmethod + @replace_list_option_in_docstrings(FEATURE_EXTRACTOR_MAPPING_NAMES) + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" + Instantiate one of the feature extractor classes of the library from a pretrained model vocabulary. + + The feature extractor class to instantiate is selected based on the `model_type` property of the config object + (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's + missing, by falling back to using pattern matching on `pretrained_model_name_or_path`: + + List options + + Params: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a feature extractor file saved using the + [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] method, e.g., + `./my_model_directory/`. + - a path or url to a saved feature extractor JSON *file*, e.g., + `./my_model_directory/preprocessor_config.json`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model feature extractor should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the feature extractor files and override the cached versions + if they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + If `False`, then this function returns just the final feature extractor object. If `True`, then this + functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary + consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of + `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored. + trust_remote_code (`bool`, *optional*, defaults to `False`): + Whether or not to allow for custom models defined on the Hub in their own modeling files. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + kwargs (`Dict[str, Any]`, *optional*): + The values in kwargs of any keys which are feature extractor attributes will be used to override the + loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is + controlled by the `return_unused_kwargs` keyword parameter. + + + + Passing `token=True` is required when you want to use a private model. + + + + Examples: + + ```python + >>> from transformers import AutoFeatureExtractor + + >>> # Download feature extractor from huggingface.co and cache. + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h") + + >>> # If feature extractor files are in a directory (e.g. feature extractor was saved using *save_pretrained('./test/saved_model/')*) + >>> # feature_extractor = AutoFeatureExtractor.from_pretrained("./test/saved_model/") + ```""" + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if kwargs.get("token", None) is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + kwargs["token"] = use_auth_token + + config = kwargs.pop("config", None) + trust_remote_code = kwargs.pop("trust_remote_code", None) + kwargs["_from_auto"] = True + + config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs) + feature_extractor_class = config_dict.get("feature_extractor_type", None) + feature_extractor_auto_map = None + if "AutoFeatureExtractor" in config_dict.get("auto_map", {}): + feature_extractor_auto_map = config_dict["auto_map"]["AutoFeatureExtractor"] + + # If we don't find the feature extractor class in the feature extractor config, let's try the model config. + if feature_extractor_class is None and feature_extractor_auto_map is None: + if not isinstance(config, PretrainedConfig): + config = AutoConfig.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs + ) + # It could be in `config.feature_extractor_type`` + feature_extractor_class = getattr(config, "feature_extractor_type", None) + if hasattr(config, "auto_map") and "AutoFeatureExtractor" in config.auto_map: + feature_extractor_auto_map = config.auto_map["AutoFeatureExtractor"] + + if feature_extractor_class is not None: + feature_extractor_class = feature_extractor_class_from_name(feature_extractor_class) + + has_remote_code = feature_extractor_auto_map is not None + has_local_code = feature_extractor_class is not None or type(config) in FEATURE_EXTRACTOR_MAPPING + trust_remote_code = resolve_trust_remote_code( + trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code + ) + + if has_remote_code and trust_remote_code: + feature_extractor_class = get_class_from_dynamic_module( + feature_extractor_auto_map, pretrained_model_name_or_path, **kwargs + ) + _ = kwargs.pop("code_revision", None) + if os.path.isdir(pretrained_model_name_or_path): + feature_extractor_class.register_for_auto_class() + return feature_extractor_class.from_dict(config_dict, **kwargs) + elif feature_extractor_class is not None: + return feature_extractor_class.from_dict(config_dict, **kwargs) + # Last try: we use the FEATURE_EXTRACTOR_MAPPING. + elif type(config) in FEATURE_EXTRACTOR_MAPPING: + feature_extractor_class = FEATURE_EXTRACTOR_MAPPING[type(config)] + return feature_extractor_class.from_dict(config_dict, **kwargs) + + raise ValueError( + f"Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a " + f"`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following " + f"`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys())}" + ) + + @staticmethod + def register(config_class, feature_extractor_class, exist_ok=False): + """ + Register a new feature extractor for this class. + + Args: + config_class ([`PretrainedConfig`]): + The configuration corresponding to the model to register. + feature_extractor_class ([`FeatureExtractorMixin`]): The feature extractor to register. + """ + FEATURE_EXTRACTOR_MAPPING.register(config_class, feature_extractor_class, exist_ok=exist_ok) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_sequence_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_sequence_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c9a26bac9b3dcd9bb14d855f34494a38df3f7f71 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_sequence_utils.py @@ -0,0 +1,371 @@ +# Copyright 2021 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Sequence feature extraction class for common feature extractors to preprocess sequences. +""" + +from typing import Optional, Union + +import numpy as np + +from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin +from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy + + +logger = logging.get_logger(__name__) + + +class SequenceFeatureExtractor(FeatureExtractionMixin): + """ + This is a general feature extraction class for speech recognition. + + Args: + feature_size (`int`): + The feature dimension of the extracted features. + sampling_rate (`int`): + The sampling rate at which the audio files should be digitalized expressed in hertz (Hz). + padding_value (`float`): + The value that is used to fill the padding values / vectors. + """ + + def __init__(self, feature_size: int, sampling_rate: int, padding_value: float, **kwargs): + self.feature_size = feature_size + self.sampling_rate = sampling_rate + self.padding_value = padding_value + + self.padding_side = kwargs.pop("padding_side", "right") + self.return_attention_mask = kwargs.pop("return_attention_mask", True) + + super().__init__(**kwargs) + + def pad( + self, + processed_features: Union[ + BatchFeature, + list[BatchFeature], + dict[str, BatchFeature], + dict[str, list[BatchFeature]], + list[dict[str, BatchFeature]], + ], + padding: Union[bool, str, PaddingStrategy] = True, + max_length: Optional[int] = None, + truncation: bool = False, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + ) -> BatchFeature: + """ + Pad input values / input vectors or a batch of input values / input vectors up to predefined length or to the + max sequence length in the batch. + + Padding side (left/right) padding values are defined at the feature extractor level (with `self.padding_side`, + `self.padding_value`) + + + + If the `processed_features` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the + result will use the same type unless you provide a different tensor type with `return_tensors`. In the case of + PyTorch tensors, you will lose the specific device of your tensors however. + + + + Args: + processed_features ([`BatchFeature`], list of [`BatchFeature`], `Dict[str, List[float]]`, `Dict[str, List[List[float]]` or `List[Dict[str, List[float]]]`): + Processed inputs. Can represent one input ([`BatchFeature`] or `Dict[str, List[float]]`) or a batch of + input values / vectors (list of [`BatchFeature`], *Dict[str, List[List[float]]]* or *List[Dict[str, + List[float]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader + collate function. + + Instead of `List[float]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors), + see the note above for the return type. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding + index) among: + + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + truncation (`bool`): + Activates truncation to cut input sequences longer than `max_length` to `max_length`. + pad_to_multiple_of (`int`, *optional*): + If set will pad the sequence to a multiple of the provided value. + + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability + `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. + return_attention_mask (`bool`, *optional*): + Whether to return the attention mask. If left to the default, will return the attention mask according + to the specific feature_extractor's default. + + [What are attention masks?](../glossary#attention-mask) + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors instead of list of python integers. Acceptable values are: + + - `'tf'`: Return TensorFlow `tf.constant` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return Numpy `np.ndarray` objects. + """ + # If we have a list of dicts, let's convert it in a dict of lists + # We do this to allow using this method as a collate_fn function in PyTorch Dataloader + if isinstance(processed_features, (list, tuple)) and isinstance(processed_features[0], (dict, BatchFeature)): + processed_features = { + key: [example[key] for example in processed_features] for key in processed_features[0].keys() + } + + # The model's main input name, usually `input_values`, has be passed for padding + if self.model_input_names[0] not in processed_features: + raise ValueError( + "You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`" + f" to this method that includes {self.model_input_names[0]}, but you provided" + f" {list(processed_features.keys())}" + ) + + required_input = processed_features[self.model_input_names[0]] + return_attention_mask = ( + return_attention_mask if return_attention_mask is not None else self.return_attention_mask + ) + + if len(required_input) == 0: + if return_attention_mask: + processed_features["attention_mask"] = [] + return processed_features + + # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays + # and rebuild them afterwards if no return_tensors is specified + # Note that we lose the specific device the tensor may be on for PyTorch + + first_element = required_input[0] + if isinstance(first_element, (list, tuple)): + # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element. + index = 0 + while len(required_input[index]) == 0: + index += 1 + if index < len(required_input): + first_element = required_input[index][0] + + if return_tensors is None: + if is_tf_tensor(first_element): + return_tensors = "tf" + elif is_torch_tensor(first_element): + return_tensors = "pt" + elif isinstance(first_element, (int, float, list, tuple, np.ndarray)): + return_tensors = "np" + else: + raise ValueError( + f"type of {first_element} unknown: {type(first_element)}. " + "Should be one of a python, numpy, pytorch or tensorflow object." + ) + + for key, value in processed_features.items(): + if isinstance(value[0], (int, float)): + processed_features[key] = to_numpy(value) + else: + processed_features[key] = [to_numpy(v) for v in value] + + # Convert padding_strategy in PaddingStrategy + padding_strategy = self._get_padding_strategies(padding=padding, max_length=max_length) + + required_input = processed_features[self.model_input_names[0]] + + batch_size = len(required_input) + if not all(len(v) == batch_size for v in processed_features.values()): + raise ValueError("Some items in the output dictionary have a different batch size than others.") + + truncated_inputs = [] + for i in range(batch_size): + inputs = {k: v[i] for k, v in processed_features.items()} + # truncation + inputs_slice = self._truncate( + inputs, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + truncation=truncation, + ) + truncated_inputs.append(inputs_slice) + + if padding_strategy == PaddingStrategy.LONGEST: + # make sure that `max_length` cannot be longer than the longest truncated length + max_length = max(len(input_slice[self.model_input_names[0]]) for input_slice in truncated_inputs) + padding_strategy = PaddingStrategy.MAX_LENGTH + + batch_outputs = {} + for i in range(batch_size): + # padding + outputs = self._pad( + truncated_inputs[i], + max_length=max_length, + padding_strategy=padding_strategy, + pad_to_multiple_of=pad_to_multiple_of, + return_attention_mask=return_attention_mask, + ) + + for key, value in outputs.items(): + if key not in batch_outputs: + batch_outputs[key] = [] + if value.dtype is np.dtype(np.float64): + value = value.astype(np.float32) + batch_outputs[key].append(value) + + return BatchFeature(batch_outputs, tensor_type=return_tensors) + + def _pad( + self, + processed_features: Union[dict[str, np.ndarray], BatchFeature], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + ) -> dict: + """ + Pad inputs (on left/right and up to predefined length or max length in the batch) + + Args: + processed_features (`Union[Dict[str, np.ndarray], BatchFeature]`): + Dictionary of input values (`np.ndarray[float]`) / input vectors (`List[np.ndarray[float]]`) or batch + of inputs values (`List[np.ndarray[int]]`) / input vectors (`List[np.ndarray[int]]`) + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see below) + padding_strategy (`PaddingStrategy`, *optional*, default to `PaddingStrategy.DO_NOT_PAD`): + PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The feature_extractor padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of (`int`, *optional*): + Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to + enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs + which benefit from having sequence lengths be a multiple of 128. + return_attention_mask (`bool`, *optional*): + Set to False to avoid returning attention mask (default: set to model specifics) + """ + required_input = processed_features[self.model_input_names[0]] + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(required_input) + + if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0): + max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) < max_length + + if return_attention_mask and "attention_mask" not in processed_features: + processed_features["attention_mask"] = np.ones(len(required_input), dtype=np.int32) + + if needs_to_be_padded: + difference = max_length - len(required_input) + if self.padding_side == "right": + if return_attention_mask: + processed_features["attention_mask"] = np.pad( + processed_features["attention_mask"], (0, difference) + ) + padding_shape = ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference) + processed_features[self.model_input_names[0]] = np.pad( + required_input, padding_shape, "constant", constant_values=self.padding_value + ) + elif self.padding_side == "left": + if return_attention_mask: + processed_features["attention_mask"] = np.pad( + processed_features["attention_mask"], (difference, 0) + ) + padding_shape = ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0) + processed_features[self.model_input_names[0]] = np.pad( + required_input, padding_shape, "constant", constant_values=self.padding_value + ) + else: + raise ValueError("Invalid padding strategy:" + str(self.padding_side)) + + return processed_features + + def _truncate( + self, + processed_features: Union[dict[str, np.ndarray], BatchFeature], + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + truncation: Optional[bool] = None, + ): + """ + Truncate inputs to predefined length or max length in the batch + + Args: + processed_features(`Union[Dict[str, np.ndarray], BatchFeature]`): + Dictionary of input values (`np.ndarray[float]`) / input vectors (`List[np.ndarray[float]]`) or batch + of inputs values (`List[np.ndarray[int]]`) / input vectors (`List[np.ndarray[int]]`) + max_length (`int`, *optional*): + maximum length of the returned list and optionally padding length (see below) + pad_to_multiple_of (`int`, *optional*) : + Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to + enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs + which benefit from having sequence lengths be a multiple of 128. + truncation (`bool`, *optional*): + Activates truncation to cut input sequences longer than `max_length` to `max_length`. + """ + if not truncation: + return processed_features + elif truncation and max_length is None: + raise ValueError("When setting ``truncation=True``, make sure that ``max_length`` is defined.") + + required_input = processed_features[self.model_input_names[0]] + + # find `max_length` that fits `pad_to_multiple_of` + if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0): + max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_truncated = len(required_input) > max_length + + if needs_to_be_truncated: + processed_features[self.model_input_names[0]] = processed_features[self.model_input_names[0]][:max_length] + if "attention_mask" in processed_features: + processed_features["attention_mask"] = processed_features["attention_mask"][:max_length] + + return processed_features + + def _get_padding_strategies(self, padding=False, max_length=None): + """ + Find the correct padding strategy + """ + + # Get padding strategy + if padding is not False: + if padding is True: + padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch + elif not isinstance(padding, PaddingStrategy): + padding_strategy = PaddingStrategy(padding) + elif isinstance(padding, PaddingStrategy): + padding_strategy = padding + else: + padding_strategy = PaddingStrategy.DO_NOT_PAD + + # Set max length if needed + if max_length is None: + if padding_strategy == PaddingStrategy.MAX_LENGTH: + raise ValueError( + f"When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined" + ) + + # Test if we have a padding value + if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None): + raise ValueError( + "Asking to pad but the feature_extractor does not have a padding value. Please select a value to use" + " as `padding_value`. For example: `feature_extractor.padding_value = 0.0`." + ) + + return padding_strategy diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ca2a3b5fde31d81554c76e26a24ebb4b806ed052 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/feature_extraction_utils.py @@ -0,0 +1,701 @@ +# Copyright 2021 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Feature extraction saving/loading class for common feature extractors. +""" + +import copy +import json +import os +import warnings +from collections import UserDict +from typing import TYPE_CHECKING, Any, Optional, Union + +import numpy as np + +from .dynamic_module_utils import custom_object_save +from .utils import ( + FEATURE_EXTRACTOR_NAME, + PushToHubMixin, + TensorType, + add_model_info_to_auto_map, + add_model_info_to_custom_pipelines, + cached_file, + copy_func, + download_url, + is_flax_available, + is_jax_tensor, + is_numpy_array, + is_offline_mode, + is_remote_url, + is_tf_available, + is_torch_available, + is_torch_device, + is_torch_dtype, + logging, + requires_backends, +) + + +if TYPE_CHECKING: + if is_torch_available(): + import torch # noqa + + +logger = logging.get_logger(__name__) + +PreTrainedFeatureExtractor = Union["SequenceFeatureExtractor"] # noqa: F821 + + +class BatchFeature(UserDict): + r""" + Holds the output of the [`~SequenceFeatureExtractor.pad`] and feature extractor specific `__call__` methods. + + This class is derived from a python dictionary and can be used as a dictionary. + + Args: + data (`dict`, *optional*): + Dictionary of lists/arrays/tensors returned by the __call__/pad methods ('input_values', 'attention_mask', + etc.). + tensor_type (`Union[None, str, TensorType]`, *optional*): + You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at + initialization. + """ + + def __init__(self, data: Optional[dict[str, Any]] = None, tensor_type: Union[None, str, TensorType] = None): + super().__init__(data) + self.convert_to_tensors(tensor_type=tensor_type) + + def __getitem__(self, item: str) -> Union[Any]: + """ + If the key is a string, returns the value of the dict associated to `key` ('input_values', 'attention_mask', + etc.). + """ + if isinstance(item, str): + return self.data[item] + else: + raise KeyError("Indexing with integers is not available when using Python based feature extractors") + + def __getattr__(self, item: str): + try: + return self.data[item] + except KeyError: + raise AttributeError + + def __getstate__(self): + return {"data": self.data} + + def __setstate__(self, state): + if "data" in state: + self.data = state["data"] + + # Copied from transformers.tokenization_utils_base.BatchEncoding.keys + def keys(self): + return self.data.keys() + + # Copied from transformers.tokenization_utils_base.BatchEncoding.values + def values(self): + return self.data.values() + + # Copied from transformers.tokenization_utils_base.BatchEncoding.items + def items(self): + return self.data.items() + + def _get_is_as_tensor_fns(self, tensor_type: Optional[Union[str, TensorType]] = None): + if tensor_type is None: + return None, None + + # Convert to TensorType + if not isinstance(tensor_type, TensorType): + tensor_type = TensorType(tensor_type) + + # Get a function reference for the correct framework + if tensor_type == TensorType.TENSORFLOW: + if not is_tf_available(): + raise ImportError( + "Unable to convert output to TensorFlow tensors format, TensorFlow is not installed." + ) + import tensorflow as tf + + as_tensor = tf.constant + is_tensor = tf.is_tensor + elif tensor_type == TensorType.PYTORCH: + if not is_torch_available(): + raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.") + import torch # noqa + + def as_tensor(value): + if isinstance(value, (list, tuple)) and len(value) > 0: + if isinstance(value[0], np.ndarray): + value = np.array(value) + elif ( + isinstance(value[0], (list, tuple)) + and len(value[0]) > 0 + and isinstance(value[0][0], np.ndarray) + ): + value = np.array(value) + if isinstance(value, np.ndarray): + return torch.from_numpy(value) + else: + return torch.tensor(value) + + is_tensor = torch.is_tensor + elif tensor_type == TensorType.JAX: + if not is_flax_available(): + raise ImportError("Unable to convert output to JAX tensors format, JAX is not installed.") + import jax.numpy as jnp # noqa: F811 + + as_tensor = jnp.array + is_tensor = is_jax_tensor + else: + + def as_tensor(value, dtype=None): + if isinstance(value, (list, tuple)) and isinstance(value[0], (list, tuple, np.ndarray)): + value_lens = [len(val) for val in value] + if len(set(value_lens)) > 1 and dtype is None: + # we have a ragged list so handle explicitly + value = as_tensor([np.asarray(val) for val in value], dtype=object) + return np.asarray(value, dtype=dtype) + + is_tensor = is_numpy_array + return is_tensor, as_tensor + + def convert_to_tensors(self, tensor_type: Optional[Union[str, TensorType]] = None): + """ + Convert the inner content to tensors. + + Args: + tensor_type (`str` or [`~utils.TensorType`], *optional*): + The type of tensors to use. If `str`, should be one of the values of the enum [`~utils.TensorType`]. If + `None`, no modification is done. + """ + if tensor_type is None: + return self + + is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type) + + # Do the tensor conversion in batch + for key, value in self.items(): + try: + if not is_tensor(value): + tensor = as_tensor(value) + + self[key] = tensor + except: # noqa E722 + if key == "overflowing_values": + raise ValueError("Unable to create tensor returning overflowing values of different lengths. ") + raise ValueError( + "Unable to create tensor, you should probably activate padding " + "with 'padding=True' to have batched tensors with the same length." + ) + + return self + + def to(self, *args, **kwargs) -> "BatchFeature": + """ + Send all values to device by calling `v.to(*args, **kwargs)` (PyTorch only). This should support casting in + different `dtypes` and sending the `BatchFeature` to a different `device`. + + Args: + args (`Tuple`): + Will be passed to the `to(...)` function of the tensors. + kwargs (`Dict`, *optional*): + Will be passed to the `to(...)` function of the tensors. + To enable asynchronous data transfer, set the `non_blocking` flag in `kwargs` (defaults to `False`). + + Returns: + [`BatchFeature`]: The same instance after modification. + """ + requires_backends(self, ["torch"]) + import torch # noqa + + new_data = {} + device = kwargs.get("device") + non_blocking = kwargs.get("non_blocking", False) + # Check if the args are a device or a dtype + if device is None and len(args) > 0: + # device should be always the first argument + arg = args[0] + if is_torch_dtype(arg): + # The first argument is a dtype + pass + elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int): + device = arg + else: + # it's something else + raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.") + # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor` + for k, v in self.items(): + # check if v is a floating point + if isinstance(v, torch.Tensor) and torch.is_floating_point(v): + # cast and send to device + new_data[k] = v.to(*args, **kwargs) + elif isinstance(v, torch.Tensor) and device is not None: + new_data[k] = v.to(device=device, non_blocking=non_blocking) + else: + new_data[k] = v + self.data = new_data + return self + + +class FeatureExtractionMixin(PushToHubMixin): + """ + This is a feature extraction mixin used to provide saving/loading functionality for sequential and image feature + extractors. + """ + + _auto_class = None + + def __init__(self, **kwargs): + """Set elements of `kwargs` as attributes.""" + # Pop "processor_class" as it should be saved as private attribute + self._processor_class = kwargs.pop("processor_class", None) + # Additional attributes without default values + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error(f"Can't set {key} with value {value} for {self}") + raise err + + def _set_processor_class(self, processor_class: str): + """Sets processor class as an attribute.""" + self._processor_class = processor_class + + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ): + r""" + Instantiate a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a feature extractor, *e.g.* a + derived class of [`SequenceFeatureExtractor`]. + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a feature extractor file saved using the + [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] method, e.g., + `./my_model_directory/`. + - a path or url to a saved feature extractor JSON *file*, e.g., + `./my_model_directory/preprocessor_config.json`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model feature extractor should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the feature extractor files and override the cached versions + if they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use + the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + + + + + To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. + + + + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + If `False`, then this function returns just the final feature extractor object. If `True`, then this + functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary + consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of + `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored. + kwargs (`Dict[str, Any]`, *optional*): + The values in kwargs of any keys which are feature extractor attributes will be used to override the + loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is + controlled by the `return_unused_kwargs` keyword parameter. + + Returns: + A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]. + + Examples: + + ```python + # We can't instantiate directly the base class *FeatureExtractionMixin* nor *SequenceFeatureExtractor* so let's show the examples on a + # derived class: *Wav2Vec2FeatureExtractor* + feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained( + "facebook/wav2vec2-base-960h" + ) # Download feature_extraction_config from huggingface.co and cache. + feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained( + "./test/saved_model/" + ) # E.g. feature_extractor (or model) was saved using *save_pretrained('./test/saved_model/')* + feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("./test/saved_model/preprocessor_config.json") + feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained( + "facebook/wav2vec2-base-960h", return_attention_mask=False, foo=False + ) + assert feature_extractor.return_attention_mask is False + feature_extractor, unused_kwargs = Wav2Vec2FeatureExtractor.from_pretrained( + "facebook/wav2vec2-base-960h", return_attention_mask=False, foo=False, return_unused_kwargs=True + ) + assert feature_extractor.return_attention_mask is False + assert unused_kwargs == {"foo": False} + ```""" + kwargs["cache_dir"] = cache_dir + kwargs["force_download"] = force_download + kwargs["local_files_only"] = local_files_only + kwargs["revision"] = revision + + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + kwargs["token"] = token + + feature_extractor_dict, kwargs = cls.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs) + + return cls.from_dict(feature_extractor_dict, **kwargs) + + def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save a feature_extractor object to the directory `save_directory`, so that it can be re-loaded using the + [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the feature extractor JSON file will be saved (will be created if it does not exist). + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + use_auth_token = kwargs.pop("use_auth_token", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if kwargs.get("token", None) is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + kwargs["token"] = use_auth_token + + if os.path.isfile(save_directory): + raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file") + + os.makedirs(save_directory, exist_ok=True) + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = self._create_repo(repo_id, **kwargs) + files_timestamps = self._get_files_timestamps(save_directory) + + # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be + # loaded from the Hub. + if self._auto_class is not None: + custom_object_save(self, save_directory, config=self) + + # If we save using the predefined names, we can load using `from_pretrained` + output_feature_extractor_file = os.path.join(save_directory, FEATURE_EXTRACTOR_NAME) + + self.to_json_file(output_feature_extractor_file) + logger.info(f"Feature extractor saved in {output_feature_extractor_file}") + + if push_to_hub: + self._upload_modified_files( + save_directory, + repo_id, + files_timestamps, + commit_message=commit_message, + token=kwargs.get("token"), + ) + + return [output_feature_extractor_file] + + @classmethod + def get_feature_extractor_dict( + cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs + ) -> tuple[dict[str, Any], dict[str, Any]]: + """ + From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating a + feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] using `from_dict`. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`): + The identifier of the pre-trained checkpoint from which we want the dictionary of parameters. + + Returns: + `Tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the feature extractor object. + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", None) + proxies = kwargs.pop("proxies", None) + subfolder = kwargs.pop("subfolder", None) + token = kwargs.pop("token", None) + use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + from_pipeline = kwargs.pop("_from_pipeline", None) + from_auto_class = kwargs.pop("_from_auto", False) + + user_agent = {"file_type": "feature extractor", "from_auto_class": from_auto_class} + if from_pipeline is not None: + user_agent["using_pipeline"] = from_pipeline + + if is_offline_mode() and not local_files_only: + logger.info("Offline mode: forcing local_files_only=True") + local_files_only = True + + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + is_local = os.path.isdir(pretrained_model_name_or_path) + if os.path.isdir(pretrained_model_name_or_path): + feature_extractor_file = os.path.join(pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME) + if os.path.isfile(pretrained_model_name_or_path): + resolved_feature_extractor_file = pretrained_model_name_or_path + is_local = True + elif is_remote_url(pretrained_model_name_or_path): + feature_extractor_file = pretrained_model_name_or_path + resolved_feature_extractor_file = download_url(pretrained_model_name_or_path) + else: + feature_extractor_file = FEATURE_EXTRACTOR_NAME + try: + # Load from local folder or from cache or download from model Hub and cache + resolved_feature_extractor_file = cached_file( + pretrained_model_name_or_path, + feature_extractor_file, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + subfolder=subfolder, + token=token, + user_agent=user_agent, + revision=revision, + ) + except OSError: + # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to + # the original exception. + raise + except Exception: + # For any other exception, we throw a generic error. + raise OSError( + f"Can't load feature extractor for '{pretrained_model_name_or_path}'. If you were trying to load" + " it from 'https://huggingface.co/models', make sure you don't have a local directory with the" + f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a" + f" directory containing a {FEATURE_EXTRACTOR_NAME} file" + ) + + try: + # Load feature_extractor dict + with open(resolved_feature_extractor_file, encoding="utf-8") as reader: + text = reader.read() + feature_extractor_dict = json.loads(text) + + except json.JSONDecodeError: + raise OSError( + f"It looks like the config file at '{resolved_feature_extractor_file}' is not a valid JSON file." + ) + + if is_local: + logger.info(f"loading configuration file {resolved_feature_extractor_file}") + else: + logger.info( + f"loading configuration file {feature_extractor_file} from cache at {resolved_feature_extractor_file}" + ) + + if not is_local: + if "auto_map" in feature_extractor_dict: + feature_extractor_dict["auto_map"] = add_model_info_to_auto_map( + feature_extractor_dict["auto_map"], pretrained_model_name_or_path + ) + if "custom_pipelines" in feature_extractor_dict: + feature_extractor_dict["custom_pipelines"] = add_model_info_to_custom_pipelines( + feature_extractor_dict["custom_pipelines"], pretrained_model_name_or_path + ) + + return feature_extractor_dict, kwargs + + @classmethod + def from_dict(cls, feature_extractor_dict: dict[str, Any], **kwargs) -> PreTrainedFeatureExtractor: + """ + Instantiates a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a Python dictionary of + parameters. + + Args: + feature_extractor_dict (`Dict[str, Any]`): + Dictionary that will be used to instantiate the feature extractor object. Such a dictionary can be + retrieved from a pretrained checkpoint by leveraging the + [`~feature_extraction_utils.FeatureExtractionMixin.to_dict`] method. + kwargs (`Dict[str, Any]`): + Additional parameters from which to initialize the feature extractor object. + + Returns: + [`~feature_extraction_utils.FeatureExtractionMixin`]: The feature extractor object instantiated from those + parameters. + """ + return_unused_kwargs = kwargs.pop("return_unused_kwargs", False) + + # Update feature_extractor with kwargs if needed + to_remove = [] + for key, value in kwargs.items(): + if key in feature_extractor_dict: + feature_extractor_dict[key] = value + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + + feature_extractor = cls(**feature_extractor_dict) + + logger.info(f"Feature extractor {feature_extractor}") + if return_unused_kwargs: + return feature_extractor, kwargs + else: + return feature_extractor + + def to_dict(self) -> dict[str, Any]: + """ + Serializes this instance to a Python dictionary. Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + output = copy.deepcopy(self.__dict__) + output["feature_extractor_type"] = self.__class__.__name__ + if "mel_filters" in output: + del output["mel_filters"] + if "window" in output: + del output["window"] + return output + + @classmethod + def from_json_file(cls, json_file: Union[str, os.PathLike]) -> PreTrainedFeatureExtractor: + """ + Instantiates a feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] from the path to + a JSON file of parameters. + + Args: + json_file (`str` or `os.PathLike`): + Path to the JSON file containing the parameters. + + Returns: + A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]: The feature_extractor + object instantiated from that JSON file. + """ + with open(json_file, encoding="utf-8") as reader: + text = reader.read() + feature_extractor_dict = json.loads(text) + return cls(**feature_extractor_dict) + + def to_json_string(self) -> str: + """ + Serializes this instance to a JSON string. + + Returns: + `str`: String containing all the attributes that make up this feature_extractor instance in JSON format. + """ + dictionary = self.to_dict() + + for key, value in dictionary.items(): + if isinstance(value, np.ndarray): + dictionary[key] = value.tolist() + + # make sure private name "_processor_class" is correctly + # saved as "processor_class" + _processor_class = dictionary.pop("_processor_class", None) + if _processor_class is not None: + dictionary["processor_class"] = _processor_class + + return json.dumps(dictionary, indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this feature_extractor instance's parameters will be saved. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + writer.write(self.to_json_string()) + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + @classmethod + def register_for_auto_class(cls, auto_class="AutoFeatureExtractor"): + """ + Register this class with a given auto class. This should only be used for custom feature extractors as the ones + in the library are already mapped with `AutoFeatureExtractor`. + + + + This API is experimental and may have some slight breaking changes in the next releases. + + + + Args: + auto_class (`str` or `type`, *optional*, defaults to `"AutoFeatureExtractor"`): + The auto class to register this new feature extractor with. + """ + if not isinstance(auto_class, str): + auto_class = auto_class.__name__ + + import transformers.models.auto as auto_module + + if not hasattr(auto_module, auto_class): + raise ValueError(f"{auto_class} is not a valid auto class.") + + cls._auto_class = auto_class + + +FeatureExtractionMixin.push_to_hub = copy_func(FeatureExtractionMixin.push_to_hub) +if FeatureExtractionMixin.push_to_hub.__doc__ is not None: + FeatureExtractionMixin.push_to_hub.__doc__ = FeatureExtractionMixin.push_to_hub.__doc__.format( + object="feature extractor", object_class="AutoFeatureExtractor", object_files="feature extractor file" + ) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/features.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/features.py new file mode 100644 index 0000000000000000000000000000000000000000..da9ca2355fef785f76f727a66b36bfb95a24ee51 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/features.py @@ -0,0 +1,749 @@ +import os +from functools import partial, reduce +from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type, Union + +import transformers + +from .. import PretrainedConfig, is_tf_available, is_torch_available +from ..utils import TF2_WEIGHTS_NAME, WEIGHTS_NAME, logging +from .config import OnnxConfig + + +if TYPE_CHECKING: + from transformers import PreTrainedModel, TFPreTrainedModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +if is_torch_available(): + from transformers.models.auto import ( + AutoModel, + AutoModelForCausalLM, + AutoModelForImageClassification, + AutoModelForImageSegmentation, + AutoModelForMaskedImageModeling, + AutoModelForMaskedLM, + AutoModelForMultipleChoice, + AutoModelForObjectDetection, + AutoModelForQuestionAnswering, + AutoModelForSemanticSegmentation, + AutoModelForSeq2SeqLM, + AutoModelForSequenceClassification, + AutoModelForSpeechSeq2Seq, + AutoModelForTokenClassification, + AutoModelForVision2Seq, + ) +if is_tf_available(): + from transformers.models.auto import ( + TFAutoModel, + TFAutoModelForCausalLM, + TFAutoModelForMaskedLM, + TFAutoModelForMultipleChoice, + TFAutoModelForQuestionAnswering, + TFAutoModelForSemanticSegmentation, + TFAutoModelForSeq2SeqLM, + TFAutoModelForSequenceClassification, + TFAutoModelForTokenClassification, + ) +if not is_torch_available() and not is_tf_available(): + logger.warning( + "The ONNX export features are only supported for PyTorch or TensorFlow. You will not be able to export models" + " without one of these libraries installed." + ) + + +def supported_features_mapping( + *supported_features: str, onnx_config_cls: Optional[str] = None +) -> Dict[str, Callable[[PretrainedConfig], OnnxConfig]]: + """ + Generate the mapping between supported the features and their corresponding OnnxConfig for a given model. + + Args: + *supported_features: The names of the supported features. + onnx_config_cls: The OnnxConfig full name corresponding to the model. + + Returns: + The dictionary mapping a feature to an OnnxConfig constructor. + """ + if onnx_config_cls is None: + raise ValueError("A OnnxConfig class must be provided") + + config_cls = transformers + for attr_name in onnx_config_cls.split("."): + config_cls = getattr(config_cls, attr_name) + mapping = {} + for feature in supported_features: + if "-with-past" in feature: + task = feature.replace("-with-past", "") + mapping[feature] = partial(config_cls.with_past, task=task) + else: + mapping[feature] = partial(config_cls.from_model_config, task=feature) + + return mapping + + +class FeaturesManager: + _TASKS_TO_AUTOMODELS = {} + _TASKS_TO_TF_AUTOMODELS = {} + if is_torch_available(): + _TASKS_TO_AUTOMODELS = { + "default": AutoModel, + "masked-lm": AutoModelForMaskedLM, + "causal-lm": AutoModelForCausalLM, + "seq2seq-lm": AutoModelForSeq2SeqLM, + "sequence-classification": AutoModelForSequenceClassification, + "token-classification": AutoModelForTokenClassification, + "multiple-choice": AutoModelForMultipleChoice, + "object-detection": AutoModelForObjectDetection, + "question-answering": AutoModelForQuestionAnswering, + "image-classification": AutoModelForImageClassification, + "image-segmentation": AutoModelForImageSegmentation, + "masked-im": AutoModelForMaskedImageModeling, + "semantic-segmentation": AutoModelForSemanticSegmentation, + "vision2seq-lm": AutoModelForVision2Seq, + "speech2seq-lm": AutoModelForSpeechSeq2Seq, + } + if is_tf_available(): + _TASKS_TO_TF_AUTOMODELS = { + "default": TFAutoModel, + "masked-lm": TFAutoModelForMaskedLM, + "causal-lm": TFAutoModelForCausalLM, + "seq2seq-lm": TFAutoModelForSeq2SeqLM, + "sequence-classification": TFAutoModelForSequenceClassification, + "token-classification": TFAutoModelForTokenClassification, + "multiple-choice": TFAutoModelForMultipleChoice, + "question-answering": TFAutoModelForQuestionAnswering, + "semantic-segmentation": TFAutoModelForSemanticSegmentation, + } + + # Set of model topologies we support associated to the features supported by each topology and the factory + _SUPPORTED_MODEL_TYPE = { + "albert": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.albert.AlbertOnnxConfig", + ), + "bart": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + "sequence-classification", + "question-answering", + onnx_config_cls="models.bart.BartOnnxConfig", + ), + # BEiT cannot be used with the masked image modeling autoclass, so this feature is excluded here + "beit": supported_features_mapping( + "default", "image-classification", onnx_config_cls="models.beit.BeitOnnxConfig" + ), + "bert": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.bert.BertOnnxConfig", + ), + "big-bird": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.big_bird.BigBirdOnnxConfig", + ), + "bigbird-pegasus": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + "sequence-classification", + "question-answering", + onnx_config_cls="models.bigbird_pegasus.BigBirdPegasusOnnxConfig", + ), + "blenderbot": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + onnx_config_cls="models.blenderbot.BlenderbotOnnxConfig", + ), + "blenderbot-small": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + onnx_config_cls="models.blenderbot_small.BlenderbotSmallOnnxConfig", + ), + "bloom": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "sequence-classification", + "token-classification", + onnx_config_cls="models.bloom.BloomOnnxConfig", + ), + "camembert": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.camembert.CamembertOnnxConfig", + ), + "clip": supported_features_mapping( + "default", + onnx_config_cls="models.clip.CLIPOnnxConfig", + ), + "codegen": supported_features_mapping( + "default", + "causal-lm", + onnx_config_cls="models.codegen.CodeGenOnnxConfig", + ), + "convbert": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.convbert.ConvBertOnnxConfig", + ), + "convnext": supported_features_mapping( + "default", + "image-classification", + onnx_config_cls="models.convnext.ConvNextOnnxConfig", + ), + "data2vec-text": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.data2vec.Data2VecTextOnnxConfig", + ), + "data2vec-vision": supported_features_mapping( + "default", + "image-classification", + # ONNX doesn't support `adaptive_avg_pool2d` yet + # "semantic-segmentation", + onnx_config_cls="models.data2vec.Data2VecVisionOnnxConfig", + ), + "deberta": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "token-classification", + "question-answering", + onnx_config_cls="models.deberta.DebertaOnnxConfig", + ), + "deberta-v2": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.deberta_v2.DebertaV2OnnxConfig", + ), + "deit": supported_features_mapping( + "default", "image-classification", onnx_config_cls="models.deit.DeiTOnnxConfig" + ), + "detr": supported_features_mapping( + "default", + "object-detection", + "image-segmentation", + onnx_config_cls="models.detr.DetrOnnxConfig", + ), + "distilbert": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.distilbert.DistilBertOnnxConfig", + ), + "electra": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.electra.ElectraOnnxConfig", + ), + "flaubert": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.flaubert.FlaubertOnnxConfig", + ), + "gpt2": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "sequence-classification", + "token-classification", + onnx_config_cls="models.gpt2.GPT2OnnxConfig", + ), + "gptj": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "question-answering", + "sequence-classification", + onnx_config_cls="models.gptj.GPTJOnnxConfig", + ), + "gpt-neo": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "sequence-classification", + onnx_config_cls="models.gpt_neo.GPTNeoOnnxConfig", + ), + "groupvit": supported_features_mapping( + "default", + onnx_config_cls="models.groupvit.GroupViTOnnxConfig", + ), + "ibert": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.ibert.IBertOnnxConfig", + ), + "imagegpt": supported_features_mapping( + "default", "image-classification", onnx_config_cls="models.imagegpt.ImageGPTOnnxConfig" + ), + "layoutlm": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "token-classification", + onnx_config_cls="models.layoutlm.LayoutLMOnnxConfig", + ), + "layoutlmv3": supported_features_mapping( + "default", + "question-answering", + "sequence-classification", + "token-classification", + onnx_config_cls="models.layoutlmv3.LayoutLMv3OnnxConfig", + ), + "levit": supported_features_mapping( + "default", "image-classification", onnx_config_cls="models.levit.LevitOnnxConfig" + ), + "longt5": supported_features_mapping( + "default", + "default-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + onnx_config_cls="models.longt5.LongT5OnnxConfig", + ), + "longformer": supported_features_mapping( + "default", + "masked-lm", + "multiple-choice", + "question-answering", + "sequence-classification", + "token-classification", + onnx_config_cls="models.longformer.LongformerOnnxConfig", + ), + "marian": supported_features_mapping( + "default", + "default-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + "causal-lm", + "causal-lm-with-past", + onnx_config_cls="models.marian.MarianOnnxConfig", + ), + "mbart": supported_features_mapping( + "default", + "default-with-past", + "causal-lm", + "causal-lm-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + "sequence-classification", + "question-answering", + onnx_config_cls="models.mbart.MBartOnnxConfig", + ), + "mobilebert": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.mobilebert.MobileBertOnnxConfig", + ), + "mobilenet-v1": supported_features_mapping( + "default", + "image-classification", + onnx_config_cls="models.mobilenet_v1.MobileNetV1OnnxConfig", + ), + "mobilenet-v2": supported_features_mapping( + "default", + "image-classification", + onnx_config_cls="models.mobilenet_v2.MobileNetV2OnnxConfig", + ), + "mobilevit": supported_features_mapping( + "default", + "image-classification", + onnx_config_cls="models.mobilevit.MobileViTOnnxConfig", + ), + "mt5": supported_features_mapping( + "default", + "default-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + onnx_config_cls="models.mt5.MT5OnnxConfig", + ), + "m2m-100": supported_features_mapping( + "default", + "default-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + onnx_config_cls="models.m2m_100.M2M100OnnxConfig", + ), + "owlvit": supported_features_mapping( + "default", + onnx_config_cls="models.owlvit.OwlViTOnnxConfig", + ), + "perceiver": supported_features_mapping( + "image-classification", + "masked-lm", + "sequence-classification", + onnx_config_cls="models.perceiver.PerceiverOnnxConfig", + ), + "poolformer": supported_features_mapping( + "default", "image-classification", onnx_config_cls="models.poolformer.PoolFormerOnnxConfig" + ), + "rembert": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.rembert.RemBertOnnxConfig", + ), + "resnet": supported_features_mapping( + "default", + "image-classification", + onnx_config_cls="models.resnet.ResNetOnnxConfig", + ), + "roberta": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.roberta.RobertaOnnxConfig", + ), + "roformer": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "token-classification", + "multiple-choice", + "question-answering", + "token-classification", + onnx_config_cls="models.roformer.RoFormerOnnxConfig", + ), + "segformer": supported_features_mapping( + "default", + "image-classification", + "semantic-segmentation", + onnx_config_cls="models.segformer.SegformerOnnxConfig", + ), + "squeezebert": supported_features_mapping( + "default", + "masked-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.squeezebert.SqueezeBertOnnxConfig", + ), + "swin": supported_features_mapping( + "default", "image-classification", onnx_config_cls="models.swin.SwinOnnxConfig" + ), + "t5": supported_features_mapping( + "default", + "default-with-past", + "seq2seq-lm", + "seq2seq-lm-with-past", + onnx_config_cls="models.t5.T5OnnxConfig", + ), + "vision-encoder-decoder": supported_features_mapping( + "vision2seq-lm", onnx_config_cls="models.vision_encoder_decoder.VisionEncoderDecoderOnnxConfig" + ), + "vit": supported_features_mapping( + "default", "image-classification", onnx_config_cls="models.vit.ViTOnnxConfig" + ), + "whisper": supported_features_mapping( + "default", + "default-with-past", + "speech2seq-lm", + "speech2seq-lm-with-past", + onnx_config_cls="models.whisper.WhisperOnnxConfig", + ), + "xlm": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.xlm.XLMOnnxConfig", + ), + "xlm-roberta": supported_features_mapping( + "default", + "masked-lm", + "causal-lm", + "sequence-classification", + "multiple-choice", + "token-classification", + "question-answering", + onnx_config_cls="models.xlm_roberta.XLMRobertaOnnxConfig", + ), + "yolos": supported_features_mapping( + "default", + "object-detection", + onnx_config_cls="models.yolos.YolosOnnxConfig", + ), + } + + AVAILABLE_FEATURES = sorted(reduce(lambda s1, s2: s1 | s2, (v.keys() for v in _SUPPORTED_MODEL_TYPE.values()))) + + @staticmethod + def get_supported_features_for_model_type( + model_type: str, model_name: Optional[str] = None + ) -> Dict[str, Callable[[PretrainedConfig], OnnxConfig]]: + """ + Tries to retrieve the feature -> OnnxConfig constructor map from the model type. + + Args: + model_type (`str`): + The model type to retrieve the supported features for. + model_name (`str`, *optional*): + The name attribute of the model object, only used for the exception message. + + Returns: + The dictionary mapping each feature to a corresponding OnnxConfig constructor. + """ + model_type = model_type.lower() + if model_type not in FeaturesManager._SUPPORTED_MODEL_TYPE: + model_type_and_model_name = f"{model_type} ({model_name})" if model_name else model_type + raise KeyError( + f"{model_type_and_model_name} is not supported yet. " + f"Only {list(FeaturesManager._SUPPORTED_MODEL_TYPE.keys())} are supported. " + f"If you want to support {model_type} please propose a PR or open up an issue." + ) + return FeaturesManager._SUPPORTED_MODEL_TYPE[model_type] + + @staticmethod + def feature_to_task(feature: str) -> str: + return feature.replace("-with-past", "") + + @staticmethod + def _validate_framework_choice(framework: str): + """ + Validates if the framework requested for the export is both correct and available, otherwise throws an + exception. + """ + if framework not in ["pt", "tf"]: + raise ValueError( + f"Only two frameworks are supported for ONNX export: pt or tf, but {framework} was provided." + ) + elif framework == "pt" and not is_torch_available(): + raise RuntimeError("Cannot export model to ONNX using PyTorch because no PyTorch package was found.") + elif framework == "tf" and not is_tf_available(): + raise RuntimeError("Cannot export model to ONNX using TensorFlow because no TensorFlow package was found.") + + @staticmethod + def get_model_class_for_feature(feature: str, framework: str = "pt") -> Type: + """ + Attempts to retrieve an AutoModel class from a feature name. + + Args: + feature (`str`): + The feature required. + framework (`str`, *optional*, defaults to `"pt"`): + The framework to use for the export. + + Returns: + The AutoModel class corresponding to the feature. + """ + task = FeaturesManager.feature_to_task(feature) + FeaturesManager._validate_framework_choice(framework) + if framework == "pt": + task_to_automodel = FeaturesManager._TASKS_TO_AUTOMODELS + else: + task_to_automodel = FeaturesManager._TASKS_TO_TF_AUTOMODELS + if task not in task_to_automodel: + raise KeyError( + f"Unknown task: {feature}. Possible values are {list(FeaturesManager._TASKS_TO_AUTOMODELS.values())}" + ) + + return task_to_automodel[task] + + @staticmethod + def determine_framework(model: str, framework: Optional[str] = None) -> str: + """ + Determines the framework to use for the export. + + The priority is in the following order: + 1. User input via `framework`. + 2. If local checkpoint is provided, use the same framework as the checkpoint. + 3. Available framework in environment, with priority given to PyTorch + + Args: + model (`str`): + The name of the model to export. + framework (`str`, *optional*, defaults to `None`): + The framework to use for the export. See above for priority if none provided. + + Returns: + The framework to use for the export. + + """ + if framework is not None: + return framework + + framework_map = {"pt": "PyTorch", "tf": "TensorFlow"} + exporter_map = {"pt": "torch", "tf": "tf2onnx"} + + if os.path.isdir(model): + if os.path.isfile(os.path.join(model, WEIGHTS_NAME)): + framework = "pt" + elif os.path.isfile(os.path.join(model, TF2_WEIGHTS_NAME)): + framework = "tf" + else: + raise FileNotFoundError( + "Cannot determine framework from given checkpoint location." + f" There should be a {WEIGHTS_NAME} for PyTorch" + f" or {TF2_WEIGHTS_NAME} for TensorFlow." + ) + logger.info(f"Local {framework_map[framework]} model found.") + else: + if is_torch_available(): + framework = "pt" + elif is_tf_available(): + framework = "tf" + else: + raise EnvironmentError("Neither PyTorch nor TensorFlow found in environment. Cannot export to ONNX.") + + logger.info(f"Framework not requested. Using {exporter_map[framework]} to export to ONNX.") + + return framework + + @staticmethod + def get_model_from_feature( + feature: str, model: str, framework: Optional[str] = None, cache_dir: Optional[str] = None + ) -> Union["PreTrainedModel", "TFPreTrainedModel"]: + """ + Attempts to retrieve a model from a model's name and the feature to be enabled. + + Args: + feature (`str`): + The feature required. + model (`str`): + The name of the model to export. + framework (`str`, *optional*, defaults to `None`): + The framework to use for the export. See `FeaturesManager.determine_framework` for the priority should + none be provided. + + Returns: + The instance of the model. + + """ + framework = FeaturesManager.determine_framework(model, framework) + model_class = FeaturesManager.get_model_class_for_feature(feature, framework) + try: + model = model_class.from_pretrained(model, cache_dir=cache_dir) + except OSError: + if framework == "pt": + logger.info("Loading TensorFlow model in PyTorch before exporting to ONNX.") + model = model_class.from_pretrained(model, from_tf=True, cache_dir=cache_dir) + else: + logger.info("Loading PyTorch model in TensorFlow before exporting to ONNX.") + model = model_class.from_pretrained(model, from_pt=True, cache_dir=cache_dir) + return model + + @staticmethod + def check_supported_model_or_raise( + model: Union["PreTrainedModel", "TFPreTrainedModel"], feature: str = "default" + ) -> Tuple[str, Callable]: + """ + Check whether or not the model has the requested features. + + Args: + model: The model to export. + feature: The name of the feature to check if it is available. + + Returns: + (str) The type of the model (OnnxConfig) The OnnxConfig instance holding the model export properties. + + """ + model_type = model.config.model_type.replace("_", "-") + model_name = getattr(model, "name", "") + model_features = FeaturesManager.get_supported_features_for_model_type(model_type, model_name=model_name) + if feature not in model_features: + raise ValueError( + f"{model.config.model_type} doesn't support feature {feature}. Supported values are: {model_features}" + ) + + return model.config.model_type, FeaturesManager._SUPPORTED_MODEL_TYPE[model_type][feature] + + def get_config(model_type: str, feature: str) -> OnnxConfig: + """ + Gets the OnnxConfig for a model_type and feature combination. + + Args: + model_type (`str`): + The model type to retrieve the config for. + feature (`str`): + The feature to retrieve the config for. + + Returns: + `OnnxConfig`: config for the combination + """ + return FeaturesManager._SUPPORTED_MODEL_TYPE[model_type][feature] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/file_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ac6b36d2dbcf7fef3621a3b35f083ae89a39b1b4 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/file_utils.py @@ -0,0 +1,131 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +File utilities: utilities related to download and cache models + +This module should not be update anymore and is only left for backward compatibility. +""" + +from huggingface_hub import get_full_repo_name # for backward compatibility +from huggingface_hub.constants import HF_HUB_DISABLE_TELEMETRY as DISABLE_TELEMETRY # for backward compatibility + +from . import __version__ + +# Backward compatibility imports, to make sure all those objects can be found in file_utils +from .utils import ( + CLOUDFRONT_DISTRIB_PREFIX, + CONFIG_NAME, + DUMMY_INPUTS, + DUMMY_MASK, + ENV_VARS_TRUE_AND_AUTO_VALUES, + ENV_VARS_TRUE_VALUES, + FEATURE_EXTRACTOR_NAME, + FLAX_WEIGHTS_NAME, + HF_MODULES_CACHE, + HUGGINGFACE_CO_PREFIX, + HUGGINGFACE_CO_RESOLVE_ENDPOINT, + MODEL_CARD_NAME, + MULTIPLE_CHOICE_DUMMY_INPUTS, + PYTORCH_PRETRAINED_BERT_CACHE, + PYTORCH_TRANSFORMERS_CACHE, + S3_BUCKET_PREFIX, + SENTENCEPIECE_UNDERLINE, + SPIECE_UNDERLINE, + TF2_WEIGHTS_NAME, + TF_WEIGHTS_NAME, + TORCH_FX_REQUIRED_VERSION, + TRANSFORMERS_CACHE, + TRANSFORMERS_DYNAMIC_MODULE_NAME, + USE_JAX, + USE_TF, + USE_TORCH, + WEIGHTS_INDEX_NAME, + WEIGHTS_NAME, + ContextManagers, + DummyObject, + EntryNotFoundError, + ExplicitEnum, + ModelOutput, + PaddingStrategy, + PushToHubMixin, + RepositoryNotFoundError, + RevisionNotFoundError, + TensorType, + _LazyModule, + add_code_sample_docstrings, + add_end_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + cached_property, + copy_func, + default_cache_path, + define_sagemaker_information, + get_torch_version, + has_file, + http_user_agent, + is_apex_available, + is_bs4_available, + is_coloredlogs_available, + is_datasets_available, + is_detectron2_available, + is_faiss_available, + is_flax_available, + is_ftfy_available, + is_g2p_en_available, + is_in_notebook, + is_ipex_available, + is_librosa_available, + is_offline_mode, + is_onnx_available, + is_pandas_available, + is_phonemizer_available, + is_protobuf_available, + is_psutil_available, + is_py3nvml_available, + is_pyctcdecode_available, + is_pytesseract_available, + is_pytorch_quantization_available, + is_rjieba_available, + is_sagemaker_dp_enabled, + is_sagemaker_mp_enabled, + is_scipy_available, + is_sentencepiece_available, + is_seqio_available, + is_sklearn_available, + is_soundfile_available, + is_spacy_available, + is_speech_available, + is_tensor, + is_tensorflow_probability_available, + is_tf2onnx_available, + is_tf_available, + is_timm_available, + is_tokenizers_available, + is_torch_available, + is_torch_bf16_available, + is_torch_cuda_available, + is_torch_fx_available, + is_torch_fx_proxy, + is_torch_mps_available, + is_torch_tf32_available, + is_torch_xla_available, + is_torchaudio_available, + is_training_run_on_sagemaker, + is_vision_available, + replace_return_docstrings, + requires_backends, + to_numpy, + to_py_obj, + torch_only_method, +) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/finegrained_fp8.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/finegrained_fp8.py new file mode 100644 index 0000000000000000000000000000000000000000..d4e472a990dcfc1226d03a12bdeb7f3bd993dc7f --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/finegrained_fp8.py @@ -0,0 +1,426 @@ +# coding=utf-8 +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Tuple + +from ..utils import is_accelerate_available, is_torch_available, logging + + +if is_torch_available(): + import torch + import torch.nn as nn + import triton + import triton.language as tl + from torch.nn import functional as F + +if is_accelerate_available(): + from accelerate import init_empty_weights + + +logger = logging.get_logger(__name__) + + +# Copied from https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/inference/kernel.py +@triton.jit +def act_quant_kernel(x_ptr, y_ptr, s_ptr, BLOCK_SIZE: tl.constexpr): + pid = tl.program_id(axis=0) + offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + x = tl.load(x_ptr + offs).to(tl.float32) + s = tl.max(tl.abs(x)) / 448.0 + y = x / s + y = y.to(y_ptr.dtype.element_ty) + tl.store(y_ptr + offs, y) + tl.store(s_ptr + pid, s) + + +def act_quant(x: torch.Tensor, block_size: int = 128) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.is_contiguous() + assert x.shape[-1] % block_size == 0 + y = torch.empty_like(x, dtype=torch.float8_e4m3fn) + s = x.new_empty(*x.size()[:-1], x.size(-1) // block_size, dtype=torch.float32) + + def grid(meta): + return (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),) + + act_quant_kernel[grid](x, y, s, BLOCK_SIZE=block_size) + return y, s + + +# Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/quantization/fp8_kernel.py +@triton.jit +def _w8a8_block_fp8_matmul( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Triton-accelerated function used to perform linear operations (dot + product) on input tensors `A` and `B` with block-wise quantization, and + store the result in output tensor `C`. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + + k_start = k * BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +def w8a8_block_fp8_matmul_triton( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise + quantization. + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should + be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + Returns: + torch.Tensor: The result of matmul. + """ + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + N, K = B.shape + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + BLOCK_SIZE_M = 128 + if M < BLOCK_SIZE_M: + BLOCK_SIZE_M = triton.next_power_of_2(M) + BLOCK_SIZE_M = max(BLOCK_SIZE_M, 16) + BLOCK_SIZE_K = block_k + assert block_k % BLOCK_SIZE_K == 0 + BLOCK_SIZE_N = block_n + + def grid(META): + return (triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]),) + + _w8a8_block_fp8_matmul[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + BLOCK_SIZE_K=BLOCK_SIZE_K, + GROUP_SIZE_M=8, + ) + + return C + + +# Python version of the above triton function, it's much slower than the triton version, for testing +@torch.compile +def w8a8_block_fp8_matmul_compile( + input_q: torch.Tensor, # [batch, seq_len, hidden_dim] + weight_q: torch.Tensor, # [out_features, hidden_dim] + input_scale: torch.Tensor, # [batch * seq_len, num_input_groups] + weight_scale: torch.Tensor, # [num_weight_blocks_m, num_weight_blocks_n] + block_size: Optional[Tuple[int, int]] = None, # (M=128, N=128) for weights for example + output_dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + """ + Performs blocked matrix multiplication with FP8 quantized matrices. + + Args: + input_q: Quantized input tensor with 1x128 block quantization + weight_q: Quantized weight tensor with 128x128 block quantization + input_scale: Scaling factors for input blocks + weight_scale: Scaling factors for weight blocks + block_size: Tuple of (M, N) for weight block dimensions + output_dtype: Desired output dtype + """ + batch_size, seq_len, hidden_dim = input_q.shape if input_q.ndim == 3 else (1, input_q.shape[0], input_q.shape[1]) + out_features = weight_q.shape[0] + + # Reshape input for batched matmul + input_reshaped = input_q.view(-1, hidden_dim) # [batch*seq_len, hidden_dim] + input_scale_reshaped = input_scale.view(input_scale.shape[0], -1) # [batch*seq_len, 1] + # Calculate number of blocks + num_weight_blocks_m = out_features // block_size[0] + num_weight_blocks_n = hidden_dim // block_size[1] + + output = torch.zeros((batch_size * seq_len, out_features), dtype=torch.float32, device=input_q.device) + + for i in range(num_weight_blocks_m): + m_start = i * block_size[0] + m_end = m_start + block_size[0] + + for j in range(num_weight_blocks_n): + n_start = j * block_size[1] + n_end = n_start + block_size[1] + + # Extract current blocks + input_block = input_reshaped[:, n_start:n_end] + weight_block = weight_q[m_start:m_end, n_start:n_end] + + # Get corresponding scales + curr_input_scale = input_scale_reshaped[:, j : j + 1] # [batch*seq_len, 1] + curr_weight_scale = weight_scale[i, j] # scalar + + block_result = ( + torch._scaled_mm( + input_block, + weight_block.t(), + scale_a=torch.tensor(1, dtype=torch.float32, device=input_q.device), + scale_b=curr_weight_scale, + out_dtype=output_dtype, + ) + * curr_input_scale + ) + + output[:, m_start:m_end] += block_result + + output = output.view(batch_size, seq_len, out_features) + + return output.to(output_dtype) + + +class FP8Linear(nn.Linear): + dtype = torch.float8_e4m3fn + + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = False, + dtype=None, + block_size: Optional[Tuple[int, int]] = None, + device=None, + activation_scheme="dynamic", + ): + super().__init__(in_features, out_features) + self.in_features = in_features + self.out_features = out_features + + self.weight = torch.nn.Parameter(torch.empty(out_features, in_features, dtype=FP8Linear.dtype, device=device)) + + if self.weight.element_size() == 1: + scale_out_features = (out_features + block_size[0] - 1) // block_size[0] + scale_in_features = (in_features + block_size[1] - 1) // block_size[1] + self.weight_scale_inv = nn.Parameter( + torch.empty(scale_out_features, scale_in_features, dtype=torch.float32, device=device) + ) + else: + self.register_parameter("weight_scale_inv", None) + + self.block_size = block_size + + self.activation_scheme = activation_scheme + + if bias: + self.bias = nn.Parameter(torch.empty(self.out_features)) + else: + self.register_parameter("bias", None) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if self.weight.element_size() > 1: + return F.linear(input, self.weight, self.bias) + else: + # Context manager used to switch among the available cuda devices + # with torch.cuda.device(input.device): + qinput, scale = act_quant(input, self.block_size[1]) + # Blocks the CPU until all CUDA operations on the specified device are complete. It is used to ensure that the results of the + # preceding operations are ready before proceeding + # torch.cuda.synchronize(device=self.weight.device) + with torch.cuda.device(input.device): + output = w8a8_block_fp8_matmul_triton( + qinput, + self.weight, + scale, + self.weight_scale_inv, + self.block_size, + output_dtype=input.dtype, + ) + torch.cuda.synchronize() + if self.bias is not None: + output = output + self.bias + return output.to(dtype=input.dtype) + + +def _replace_with_fp8_linear( + model, + tp_plan=None, + modules_to_not_convert=None, + current_key_name=None, + quantization_config=None, + has_been_replaced=False, +): + """Replace Linear layers with FP8Linear.""" + if current_key_name is None: + current_key_name = [] + + for name, module in model.named_children(): + current_key_name.append(name) + + if isinstance(module, nn.Linear) and name not in (modules_to_not_convert or []): + current_key_name_str = ".".join(current_key_name) + if not any(key in current_key_name_str for key in (modules_to_not_convert or [])): + with init_empty_weights(): + model._modules[name] = FP8Linear( + in_features=module.in_features, + out_features=module.out_features, + bias=module.bias is not None, + device=module.weight.device, + dtype=module.weight.dtype, + activation_scheme=quantization_config.activation_scheme, + block_size=quantization_config.weight_block_size, + ) + has_been_replaced = True + # when changing a layer the TP PLAN for that layer should be updated. TODO + + if len(list(module.children())) > 0: + _, has_been_replaced = _replace_with_fp8_linear( + module, + tp_plan, + modules_to_not_convert, + current_key_name, + quantization_config, + has_been_replaced=has_been_replaced, + ) + + current_key_name.pop(-1) + + return model, has_been_replaced + + +def replace_with_fp8_linear( + model, + modules_to_not_convert=None, + quantization_config=None, +): + """Helper function to replace model layers with FP8 versions.""" + modules_to_not_convert = ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert + + if quantization_config.modules_to_not_convert is not None: + modules_to_not_convert.extend(quantization_config.modules_to_not_convert) + modules_to_not_convert = list(set(modules_to_not_convert)) + model, has_been_replaced = _replace_with_fp8_linear( + model, + tp_plan=model._tp_plan, + modules_to_not_convert=modules_to_not_convert, + quantization_config=quantization_config, + ) + + if not has_been_replaced: + logger.warning( + "You are loading your model using fp8 but no linear modules were found in your model." + " Please double check your model architecture." + ) + + return model diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flash_attention.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flash_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..a78166ed040b620c6e24a7e2c64c06c96f2d89d9 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flash_attention.py @@ -0,0 +1,65 @@ +from typing import Optional, Tuple + +import torch + +from ..modeling_flash_attention_utils import _flash_attention_forward, flash_attn_supports_top_left_mask + + +_use_top_left_mask = flash_attn_supports_top_left_mask() + + +def flash_attention_forward( + module: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + dropout: float = 0.0, + scaling: Optional[float] = None, + sliding_window: Optional[int] = None, + softcap: Optional[float] = None, + **kwargs, +) -> Tuple[torch.Tensor, None]: + # This is before the transpose + seq_len = query.shape[2] + + # FA2 uses non-transposed inputs + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (usually our RMSNorm modules handle it correctly) + target_dtype = None + if query.dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(module.config, "_pre_quantization_dtype"): + target_dtype = module.config._pre_quantization_dtype + else: + target_dtype = next(layer for layer in module.modules() if isinstance(layer, torch.nn.Linear)).weight.dtype + + # FA2 always relies on the value set in the module, so remove it if present in kwargs to avoid passing it twice + kwargs.pop("is_causal", None) + + attn_output = _flash_attention_forward( + query, + key, + value, + attention_mask, + query_length=seq_len, + is_causal=module.is_causal, + dropout=dropout, + softmax_scale=scaling, + sliding_window=sliding_window, + softcap=softcap, + use_top_left_mask=_use_top_left_mask, + target_dtype=target_dtype, + **kwargs, + ) + + return attn_output, None diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flax_logits_process.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flax_logits_process.py new file mode 100644 index 0000000000000000000000000000000000000000..d106c32defa4db3577d3a72158550d355aa5ece7 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flax_logits_process.py @@ -0,0 +1,544 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect + +import jax +import jax.lax as lax +import jax.numpy as jnp +from jax.experimental import sparse + +from ..utils import add_start_docstrings +from ..utils.logging import get_logger + + +logger = get_logger(__name__) + + +LOGITS_PROCESSOR_INPUTS_DOCSTRING = r""" + Args: + input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`): + Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam + search or log softmax for each vocabulary token when using beam search + kwargs (`Dict[str, Any]`, *optional*): + Additional logits processor specific kwargs. + + Return: + `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores. + +""" + + +class FlaxLogitsProcessor: + """Abstract base class for all logit processors that can be applied during generation.""" + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray) -> jnp.ndarray: + """Flax method for processing logits.""" + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + +class FlaxLogitsWarper: + """Abstract base class for all logit warpers that can be applied during generation with multinomial sampling.""" + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray) -> jnp.ndarray: + """Flax method for warping logits.""" + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + +class FlaxLogitsProcessorList(list): + """ + This class can be used to create a list of [`FlaxLogitsProcessor`] or [`FlaxLogitsWarper`] to subsequently process + a `scores` input tensor. This class inherits from list and adds a specific *__call__* method to apply each + [`FlaxLogitsProcessor`] or [`FlaxLogitsWarper`] to the inputs. + """ + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int, **kwargs) -> jnp.ndarray: + for processor in self: + function_args = inspect.signature(processor.__call__).parameters + if len(function_args) > 3: + if not all(arg in kwargs for arg in list(function_args.keys())[2:]): + raise ValueError( + f"Make sure that all the required parameters: {list(function_args.keys())} for " + f"{processor.__class__} are passed to the logits processor." + ) + scores = processor(input_ids, scores, cur_len, **kwargs) + else: + scores = processor(input_ids, scores, cur_len) + return scores + + +class FlaxTemperatureLogitsWarper(FlaxLogitsWarper): + r""" + [`FlaxLogitsWarper`] for temperature (exponential scaling output probability distribution). + + Args: + temperature (`float`): + The value used to module the logits distribution. + """ + + def __init__(self, temperature: float): + if not isinstance(temperature, float) or not (temperature > 0): + raise ValueError(f"`temperature` has to be a strictly positive float, but is {temperature}") + + self.temperature = temperature + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + scores = scores / self.temperature + return scores + + +class FlaxTopPLogitsWarper(FlaxLogitsWarper): + """ + [`FlaxLogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off. + + Args: + top_p (`float`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + filter_value (`float`, *optional*, defaults to -inf): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + """ + + def __init__(self, top_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): + if not isinstance(top_p, float) or (top_p < 0 or top_p > 1.0): + raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}") + if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1): + raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}") + + self.top_p = top_p + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + topk_scores, topk_indices = lax.top_k(scores, scores.shape[-1]) + + mask_scores = jnp.full_like(scores, self.filter_value) + cumulative_probs = jax.nn.softmax(topk_scores, axis=-1).cumsum(axis=-1) + score_mask = cumulative_probs < self.top_p + + # include the token that is higher than top_p as well + score_mask = jnp.roll(score_mask, 1) + score_mask |= score_mask.at[:, 0].set(True) + + # min tokens to keep + score_mask = score_mask.at[:, : self.min_tokens_to_keep].set(True) + + topk_next_scores = jnp.where(score_mask, topk_scores, mask_scores) + next_scores = jax.lax.sort_key_val(topk_indices, topk_next_scores)[-1] + + return next_scores + + +class FlaxTopKLogitsWarper(FlaxLogitsWarper): + r""" + [`FlaxLogitsWarper`] that performs top-k, i.e. restricting to the k highest probability elements. + + Args: + top_k (`int`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + filter_value (`float`, *optional*, defaults to -inf): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + """ + + def __init__(self, top_k: int, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): + if not isinstance(top_k, int) or top_k <= 0: + raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}") + + self.top_k = max(top_k, min_tokens_to_keep) + self.filter_value = filter_value + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + batch_size, vocab_size = scores.shape + next_scores_flat = jnp.full(batch_size * vocab_size, self.filter_value) + + topk = min(self.top_k, scores.shape[-1]) # Safety check + topk_scores, topk_indices = lax.top_k(scores, topk) + shift = jnp.broadcast_to((jnp.arange(batch_size) * vocab_size)[:, None], (batch_size, topk)).flatten() + topk_scores_flat = topk_scores.flatten() + topk_indices_flat = topk_indices.flatten() + shift + + next_scores_flat = next_scores_flat.at[topk_indices_flat].set(topk_scores_flat) + next_scores = next_scores_flat.reshape(batch_size, vocab_size) + return next_scores + + +class FlaxForcedBOSTokenLogitsProcessor(FlaxLogitsProcessor): + r""" + [`FlaxLogitsProcessor`] that enforces the specified token as the first generated token. + + Args: + bos_token_id (`int`): + The id of the token to force as the first generated token. + """ + + def __init__(self, bos_token_id: int): + self.bos_token_id = bos_token_id + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + new_scores = jnp.full(scores.shape, -float("inf")) + + apply_penalty = 1 - jnp.bool_(cur_len - 1) + + scores = jnp.where(apply_penalty, new_scores.at[:, self.bos_token_id].set(0), scores) + + return scores + + +class FlaxForcedEOSTokenLogitsProcessor(FlaxLogitsProcessor): + r""" + [`FlaxLogitsProcessor`] that enforces the specified token as the last generated token when `max_length` is reached. + + Args: + max_length (`int`): + The maximum length of the sequence to be generated. + eos_token_id (`int`): + The id of the token to force as the last generated token when `max_length` is reached. + """ + + def __init__(self, max_length: int, eos_token_id: int): + self.max_length = max_length + self.eos_token_id = eos_token_id + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + new_scores = jnp.full(scores.shape, -float("inf")) + + apply_penalty = 1 - jnp.bool_(cur_len - self.max_length + 1) + + scores = jnp.where(apply_penalty, new_scores.at[:, self.eos_token_id].set(0), scores) + + return scores + + +class FlaxMinLengthLogitsProcessor(FlaxLogitsProcessor): + r""" + [`FlaxLogitsProcessor`] enforcing a min-length by setting EOS probability to 0. + + Args: + min_length (`int`): + The minimum length below which the score of `eos_token_id` is set to `-float("Inf")`. + eos_token_id (`int`): + The id of the *end-of-sequence* token. + """ + + def __init__(self, min_length: int, eos_token_id: int): + if not isinstance(min_length, int) or min_length < 0: + raise ValueError(f"`min_length` has to be a positive integer, but is {min_length}") + + if not isinstance(eos_token_id, int) or eos_token_id < 0: + raise ValueError(f"`eos_token_id` has to be a positive integer, but is {eos_token_id}") + + self.min_length = min_length + self.eos_token_id = eos_token_id + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + # create boolean flag to decide if min length penalty should be applied + apply_penalty = 1 - jnp.clip(cur_len - self.min_length, 0, 1) + + scores = jnp.where(apply_penalty, scores.at[:, self.eos_token_id].set(-float("inf")), scores) + + return scores + + +class FlaxSuppressTokensAtBeginLogitsProcessor(FlaxLogitsProcessor): + r""" + [`FlaxLogitsProcessor`] supressing a list of tokens as soon as the `generate` function starts generating using + `begin_index` tokens. This should ensure that the tokens defined by `begin_suppress_tokens` are not sampled at the + beginning of the generation. + + Args: + begin_suppress_tokens (`List[int]`): + Tokens to not sample. + begin_index (`int`): + Index where the tokens are suppressed. + """ + + def __init__(self, begin_suppress_tokens, begin_index): + self.begin_suppress_tokens = list(begin_suppress_tokens) + self.begin_index = begin_index + + def __call__(self, input_ids, scores, cur_len: int): + apply_penalty = 1 - jnp.bool_(cur_len - self.begin_index) + + scores = jnp.where(apply_penalty, scores.at[:, self.begin_suppress_tokens].set(-float("inf")), scores) + + return scores + + +class FlaxSuppressTokensLogitsProcessor(FlaxLogitsProcessor): + r""" + [`FlaxLogitsProcessor`] suppressing a list of tokens at each decoding step. The processor will set their log probs + to be `-inf` so they are not sampled. + + Args: + suppress_tokens (`list`): + Tokens to not sample. + """ + + def __init__(self, suppress_tokens: list): + self.suppress_tokens = list(suppress_tokens) + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + scores = scores.at[..., self.suppress_tokens].set(-float("inf")) + + return scores + + +class FlaxForceTokensLogitsProcessor(FlaxLogitsProcessor): + r""" + [`FlaxLogitsProcessor`] that takes a list of pairs of integers which indicates a mapping from generation indices to + token indices that will be forced before sampling. The processor will set their log probs to 0 and all other tokens + to `-inf` so that they are sampled at their corresponding index. + + Args: + force_token_map (`list`): + Map giving token ids and indices where they will be forced to be sampled. + """ + + def __init__(self, force_token_map): + force_token_map = dict(force_token_map) + # Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the + # index of the array corresponds to the index of the token to be forced, for XLA compatibility. + # Indexes without forced tokens will have a negative value. + force_token_array = jnp.ones((max(force_token_map.keys()) + 1), dtype=jnp.int32) * -1 + for index, token in force_token_map.items(): + if token is not None: + force_token_array = force_token_array.at[index].set(token) + self.force_token_array = jnp.int32(force_token_array) + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + def _force_token(generation_idx): + batch_size = scores.shape[0] + current_token = self.force_token_array[generation_idx] + + new_scores = jnp.ones_like(scores, dtype=scores.dtype) * -float("inf") + updates = jnp.zeros((batch_size, 1), dtype=scores.dtype) + new_scores = lax.dynamic_update_slice(new_scores, updates, (0, current_token)) + return new_scores + + scores = lax.cond( + cur_len >= self.force_token_array.shape[0], + # If the current length is geq than the length of force_token_array, the processor does nothing. + lambda: scores, + # Otherwise, it may force a certain token. + lambda: lax.cond( + self.force_token_array[cur_len] >= 0, + # Only valid (positive) tokens are forced + lambda: _force_token(cur_len), + # Otherwise, the processor does nothing. + lambda: scores, + ), + ) + return scores + + +class FlaxWhisperTimeStampLogitsProcessor(FlaxLogitsProcessor): + r""" + Whisper specific Processor. This processor can be used to force a list of tokens. The processor will set their log + probs to `inf` so that they are sampled at their corresponding index. + + Args: + generate_config (`GenerateConfig`): + The generate config used to generate the output. The following parameters are required: + eos_token_id (`int`, *optional*, defaults to 50257): + The id of the *end-of-sequence* token. + no_timestamps_token_id (`int`, *optional*, defaults to 50363): + The id of the `"<|notimestamps|>"` token. + max_initial_timestamp_index (`int`, *optional*, defaults to 1): + Used to set the maximum value of the initial timestamp. This is used to prevent the model from + predicting timestamps that are too far in the future. + """ + + def __init__(self, generate_config, model_config, decoder_input_length): + self.eos_token_id = generate_config.eos_token_id + self.no_timestamps_token_id = generate_config.no_timestamps_token_id + self.timestamp_begin = generate_config.no_timestamps_token_id + 1 + + self.begin_index = decoder_input_length + 1 + + if generate_config.is_multilingual: + # room for language token and task token + self.begin_index += 2 + if hasattr(generate_config, "max_initial_timestamp_index"): + self.max_initial_timestamp_index = generate_config.max_initial_timestamp_index + else: + self.max_initial_timestamp_index = model_config.vocab_size + if self.max_initial_timestamp_index is None: + self.max_initial_timestamp_index = model_config.vocab_size + + def __call__(self, input_ids, scores, cur_len): + # suppress <|notimestamps|> which is handled by without_timestamps + scores = scores.at[:, self.no_timestamps_token_id].set(-float("inf")) + + def handle_pairs(input_ids_k, scores_k): + last_was_timestamp = jnp.where((cur_len - self.begin_index) >= 1, True, False) + last_was_timestamp = jnp.where( + input_ids_k[cur_len - 1] >= self.timestamp_begin, + True and last_was_timestamp, + False, + ) + + penultimate_was_timestamp = jnp.where((cur_len - self.begin_index) < 2, True, False) + penultimate_was_timestamp = jnp.where( + input_ids_k[cur_len - 2] >= self.timestamp_begin, + True, + penultimate_was_timestamp, + ) + + return jnp.where( + last_was_timestamp, + jnp.where( + penultimate_was_timestamp > 0, + scores_k.at[self.timestamp_begin :].set(-float("inf")), + scores_k.at[: self.eos_token_id].set(-float("inf")), + ), + scores_k, + ) + + scores = jax.vmap(handle_pairs)(input_ids, scores) + + apply_max_initial_timestamp = jnp.where(cur_len == self.begin_index, True, False) + apply_max_initial_timestamp = jnp.where( + self.max_initial_timestamp_index is not None, + True and apply_max_initial_timestamp, + False, + ) + + last_allowed = self.timestamp_begin + self.max_initial_timestamp_index + + scores = jnp.where( + apply_max_initial_timestamp, + scores.at[:, last_allowed + 1 :].set(-float("inf")), + scores, + ) + + # if sum of probability over timestamps is above any other token, sample timestamp + logprobs = jax.nn.log_softmax(scores, axis=-1) + + def handle_cumulative_probs(logprobs_k, scores_k): + timestamp_logprob = jax.nn.logsumexp(logprobs_k[self.timestamp_begin :], axis=-1) + max_text_token_logprob = jnp.max(logprobs_k[: self.timestamp_begin]) + return jnp.where( + timestamp_logprob > max_text_token_logprob, + scores_k.at[: self.timestamp_begin].set(-float("inf")), + scores_k, + ) + + scores = jax.vmap(handle_cumulative_probs)(logprobs, scores) + + return scores + + +class FlaxNoRepeatNGramLogitsProcessor(FlaxLogitsProcessor): + r""" + [`FlaxLogitsProcessor`] that enforces no repetition of n-grams. See + [Fairseq](https://github.com/pytorch/fairseq/blob/a07cb6f40480928c9e0548b737aadd36ee66ac76/fairseq/sequence_generator.py#L345). + + Args: + ngram_size (`int`): + All ngrams of size `ngram_size` can only occur once. + """ + + def __init__(self, ngram_size: int): + if not isinstance(ngram_size, int) or ngram_size <= 0: + raise ValueError(f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}") + self.ngram_size = ngram_size + + def get_previous_ngrams(self, input_ids: jnp.ndarray, vocab_size: int, cur_len: int): + """ + get a matrix of size (batch_size,) + (vocab_size,)*n (for n-grams) that + represent the n-grams that occurred previously. + The BCOO representation allow to store only the few non-zero entries, instead of the full (huge) matrix + """ + batch_size, seq_len = input_ids.shape + # number of n-grams in the whole sequence + seq_ngrams = seq_len - (self.ngram_size - 1) + # number of n-grams in the currently generated sequence + cur_ngrams = cur_len - (self.ngram_size - 1) + + def body_fun(i, val): + b = i % batch_size + pos = i // batch_size + return val.at[i].set( + jnp.array( + [ + b, + ] + + [jnp.array(input_ids)[b, pos + j] for j in range(self.ngram_size)] + ) + ) + + shape = (batch_size * seq_ngrams, self.ngram_size + 1) + all_update_indices = jax.lax.fori_loop( + 0, batch_size * cur_ngrams, body_fun, jnp.zeros(shape, dtype=input_ids.dtype) + ) + + # ignore the n-grams not yet generated + data = (jnp.arange(batch_size * seq_ngrams) < batch_size * cur_ngrams).astype("float32") + + return sparse.BCOO((data, all_update_indices), shape=(batch_size,) + (vocab_size,) * self.ngram_size) + + def get_banned_tokens_mask(self, latest_tokens: jnp.ndarray, previous_ngrams) -> jnp.ndarray: + """ + Determines which tokens must be banned given latest tokens and the previously seen + ngrams. + """ + + @sparse.sparsify + @jax.vmap + def inner_fn(latest_tokens, previous_ngrams): + return previous_ngrams[tuple(latest_tokens)] + + return sparse.bcoo_todense(inner_fn(latest_tokens, previous_ngrams)) + + def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_len: int) -> jnp.ndarray: + def true_fn(): + _, vocab_size = scores.shape + # store the previously seen n-grams + previous_ngrams = self.get_previous_ngrams(input_ids, vocab_size, cur_len) + + # get the n-1 last tokens that prefix the n-gram being generated + latest_tokens = jnp.zeros((input_ids.shape[0], self.ngram_size - 1), dtype=input_ids.dtype) + latest_tokens = jax.lax.dynamic_update_slice( + latest_tokens, + jax.lax.dynamic_slice( + input_ids, (0, cur_len - (self.ngram_size - 1)), (input_ids.shape[0], (self.ngram_size - 1)) + ), + (0, 0), + ) + + # compute the banned tokens, ie all the tokens that when added to the latest tokens lead to a n-gram that was previously generated + banned_tokens_indices_mask = self.get_banned_tokens_mask(latest_tokens, previous_ngrams).astype("bool") + return jnp.where(banned_tokens_indices_mask, -float("inf"), scores) + + output = jax.lax.cond((cur_len >= self.ngram_size - 1), true_fn, lambda: scores) + return output diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flax_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flax_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ddd718cbb8a63646d39cc12718ca606a4b77405f --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flax_utils.py @@ -0,0 +1,1029 @@ +# coding=utf-8 +# Copyright 2021 The Google AI Flax Team Authors, and The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import copy +import inspect +import warnings +from functools import partial +from typing import Any, Dict, Optional, Union + +import flax +import jax +import jax.numpy as jnp +import numpy as np +from jax import lax + +from ..models.auto import ( + FLAX_MODEL_FOR_CAUSAL_LM_MAPPING, + FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING, + FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING, +) +from ..utils import ModelOutput, logging +from .configuration_utils import GenerationConfig +from .flax_logits_process import ( + FlaxForcedBOSTokenLogitsProcessor, + FlaxForcedEOSTokenLogitsProcessor, + FlaxForceTokensLogitsProcessor, + FlaxLogitsProcessorList, + FlaxMinLengthLogitsProcessor, + FlaxNoRepeatNGramLogitsProcessor, + FlaxSuppressTokensAtBeginLogitsProcessor, + FlaxSuppressTokensLogitsProcessor, + FlaxTemperatureLogitsWarper, + FlaxTopKLogitsWarper, + FlaxTopPLogitsWarper, +) + + +logger = logging.get_logger(__name__) + + +@flax.struct.dataclass +class FlaxGreedySearchOutput(ModelOutput): + """ + Flax Base class for outputs of decoder-only generation models using greedy search. + + + Args: + sequences (`jnp.ndarray` of shape `(batch_size, max_length)`): + The generated sequences. + """ + + sequences: Optional[jnp.ndarray] = None + + +@flax.struct.dataclass +class FlaxSampleOutput(ModelOutput): + """ + Flax Base class for outputs of decoder-only generation models using sampling. + + + Args: + sequences (`jnp.ndarray` of shape `(batch_size, max_length)`): + The generated sequences. + """ + + sequences: Optional[jnp.ndarray] = None + + +@flax.struct.dataclass +class FlaxBeamSearchOutput(ModelOutput): + """ + Flax Base class for outputs of decoder-only generation models using greedy search. + + + Args: + sequences (`jnp.ndarray` of shape `(batch_size, max_length)`): + The generated sequences. + scores (`jnp.ndarray` of shape `(batch_size,)`): + The scores (log probabilities) of the generated sequences. + """ + + sequences: Optional[jnp.ndarray] = None + scores: Optional[jnp.ndarray] = None + + +@flax.struct.dataclass +class GreedyState: + cur_len: jnp.ndarray + sequences: jnp.ndarray + running_token: jnp.ndarray + is_sent_finished: jnp.ndarray + model_kwargs: Dict[str, jnp.ndarray] + + +@flax.struct.dataclass +class SampleState: + cur_len: jnp.ndarray + sequences: jnp.ndarray + running_token: jnp.ndarray + is_sent_finished: jnp.ndarray + prng_key: jnp.ndarray + model_kwargs: Dict[str, jnp.ndarray] + + +@flax.struct.dataclass +class BeamSearchState: + cur_len: jnp.ndarray + running_sequences: jnp.ndarray + running_scores: jnp.ndarray + sequences: jnp.ndarray + scores: jnp.ndarray + is_sent_finished: jnp.ndarray + model_kwargs: Dict[str, jnp.ndarray] + + +class FlaxGenerationMixin: + """ + A class containing all functions for auto-regressive text generation, to be used as a mixin in + [`FlaxPreTrainedModel`]. + + The class exposes [`~generation.FlaxGenerationMixin.generate`], which can be used for: + - *greedy decoding* by calling [`~generation.FlaxGenerationMixin._greedy_search`] if `num_beams=1` and + `do_sample=False` + - *multinomial sampling* by calling [`~generation.FlaxGenerationMixin._sample`] if `num_beams=1` and + `do_sample=True` + - *beam-search decoding* by calling [`~generation.FlaxGenerationMixin._beam_search`] if `num_beams>1` and + `do_sample=False` + + You do not need to call any of the above methods directly. Pass custom parameter values to 'generate' instead. To + learn more about decoding strategies refer to the [text generation strategies guide](../generation_strategies). + """ + + def prepare_inputs_for_generation(self, *args, **kwargs): + raise NotImplementedError( + "A model class needs to define a `prepare_inputs_for_generation` method in order to use `generate`." + ) + + @staticmethod + def _run_loop_in_debug(cond_fn, body_fn, init_state): + """ + Run generation in untraced mode. This should only be used for debugging purposes. + """ + state = init_state + while cond_fn(state): + state = body_fn(state) + return state + + def _prepare_encoder_decoder_kwargs_for_generation(self, input_ids, params, model_kwargs): + encoder_kwargs = { + argument: value + for argument, value in model_kwargs.items() + if not (argument.startswith("decoder_") or argument.startswith("cross_attn")) + } + model_kwargs["encoder_outputs"] = self.encode(input_ids, params=params, return_dict=True, **encoder_kwargs) + return model_kwargs + + def _prepare_decoder_input_ids_for_generation( + self, + batch_size: int, + decoder_start_token_id: Optional[int] = None, + bos_token_id: Optional[int] = None, + model_kwargs: Optional[Dict[str, jnp.ndarray]] = None, + ) -> jnp.ndarray: + if model_kwargs is not None and "decoder_input_ids" in model_kwargs: + # Only use this arg if not None, otherwise just remove from model_kwargs + decoder_input_ids = model_kwargs.pop("decoder_input_ids") + if decoder_input_ids is not None: + return decoder_input_ids + decoder_start_token_id = self._get_decoder_start_token_id(decoder_start_token_id, bos_token_id) + return jnp.array(decoder_start_token_id, dtype="i4").reshape(1, -1).repeat(batch_size, axis=0) + + def _get_decoder_start_token_id( + self, decoder_start_token_id: Optional[int] = None, bos_token_id: Optional[int] = None + ) -> int: + # retrieve decoder_start_token_id for encoder-decoder models + # fall back to bos_token_id if necessary + decoder_start_token_id = ( + decoder_start_token_id + if decoder_start_token_id is not None + else self.generation_config.decoder_start_token_id + ) + bos_token_id = bos_token_id if bos_token_id is not None else self.generation_config.bos_token_id + if decoder_start_token_id is not None: + return decoder_start_token_id + elif ( + hasattr(self.config, "decoder") + and hasattr(self.config.decoder, "decoder_start_token_id") + and self.config.decoder.decoder_start_token_id is not None + ): + return self.config.decoder.decoder_start_token_id + elif bos_token_id is not None: + return bos_token_id + elif ( + hasattr(self.config, "decoder") + and hasattr(self.config.decoder, "bos_token_id") + and self.config.decoder.bos_token_id is not None + ): + return self.config.decoder.bos_token_id + raise ValueError( + "`decoder_start_token_id` or `bos_token_id` has to be defined for encoder-decoder generation." + ) + + @staticmethod + def _expand_to_num_beams(tensor, num_beams): + return jnp.broadcast_to(tensor[:, None], (tensor.shape[0], num_beams) + tensor.shape[1:]) + + def _adapt_logits_for_beam_search(self, logits): + """ + This function can be overwritten in the specific modeling_flax_.py classes to allow for custom beam + search behavior. Note that the only model that overwrites this method is [`~transformes.FlaxMarianMTModel`]. + """ + return logits + + def _validate_model_class(self): + """ + Confirms that the model class is compatible with generation. If not, raises an exception that points to the + right class to use. + """ + if not self.can_generate(): + generate_compatible_mappings = [ + FLAX_MODEL_FOR_CAUSAL_LM_MAPPING, + FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING, + FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING, + ] + generate_compatible_classes = set() + for model_mapping in generate_compatible_mappings: + supported_models = model_mapping.get(type(self.config), default=None) + if supported_models is not None: + generate_compatible_classes.add(supported_models.__name__) + exception_message = ( + f"The current model class ({self.__class__.__name__}) is not compatible with `.generate()`, as " + "it doesn't have a language model head." + ) + if generate_compatible_classes: + exception_message += f" Please use one of the following classes instead: {generate_compatible_classes}" + raise TypeError(exception_message) + + def _validate_model_kwargs(self, model_kwargs: Dict[str, Any]): + """Validates model kwargs for generation. Generate argument typos will also be caught here.""" + unused_model_args = [] + model_args = set(inspect.signature(self.prepare_inputs_for_generation).parameters) + # `kwargs`/`model_kwargs` is often used to handle optional forward pass inputs like `attention_mask`. If + # `prepare_inputs_for_generation` doesn't accept them, then a stricter check can be made ;) + if "kwargs" in model_args or "model_kwargs" in model_args: + model_args |= set(inspect.signature(self.__call__).parameters) + for key, value in model_kwargs.items(): + if value is not None and key not in model_args: + unused_model_args.append(key) + + if unused_model_args: + raise ValueError( + f"The following `model_kwargs` are not used by the model: {unused_model_args} (note: typos in the" + " generate arguments will also show up in this list)" + ) + + def generate( + self, + input_ids: jnp.ndarray, + generation_config: Optional[GenerationConfig] = None, + prng_key: Optional[jnp.ndarray] = None, + trace: bool = True, + params: Optional[Dict[str, jnp.ndarray]] = None, + logits_processor: Optional[FlaxLogitsProcessorList] = None, + **kwargs, + ): + r""" + Generates sequences of token ids for models with a language modeling head. + + Parameters: + input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): + The sequence used as a prompt for the generation. + generation_config (`~generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. `**kwargs` + passed to generate matching the attributes of `generation_config` will override them. If + `generation_config` is not provided, the default will be used, which had the following loading + priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model + configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s + default values, whose documentation should be checked to parameterize generation. + trace (`bool`, *optional*, defaults to `True`): + Whether to trace generation. Setting `trace=False` should only be used for debugging and will lead to a + considerably slower runtime. + params (`Dict[str, jnp.ndarray]`, *optional*): + Optionally the model parameters can be passed. Can be useful for parallelized generation. + logits_processor (`FlaxLogitsProcessorList `, *optional*): + Custom logits processors that complement the default logits processors built from arguments and + generation config. If a logit processor is passed that is already created with the arguments or a + generation config an error is thrown. This feature is intended for advanced users. + kwargs (`Dict[str, Any]`, *optional*): + Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be + forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder + specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*. + + Return: + [`~utils.ModelOutput`]. + + """ + # Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call + self._validate_model_class() + + # priority: `generation_config` argument > `model.generation_config` (the default generation config) + if generation_config is None: + # legacy: users may modify the model configuration to control generation. To trigger this legacy behavior, + # two conditions must be met + # 1) the generation config must have been created from the model config (`_from_model_config` field); + # 2) the generation config must have seen no modification since its creation (the hash is the same). + if self.generation_config._from_model_config and self.generation_config._original_object_hash == hash( + self.generation_config + ): + new_generation_config = GenerationConfig.from_model_config(self.config) + if new_generation_config != self.generation_config: + warnings.warn( + "You have modified the pretrained model configuration to control generation. This is a" + " deprecated strategy to control generation and will be removed soon, in a future version." + " Please use and modify the model generation configuration (see" + " https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )" + ) + self.generation_config = new_generation_config + generation_config = self.generation_config + + generation_config = copy.deepcopy(generation_config) + model_kwargs = generation_config.update(**kwargs) # All unused kwargs must be model kwargs + self._validate_model_kwargs(model_kwargs.copy()) + + logits_processor = logits_processor if logits_processor is not None else FlaxLogitsProcessorList() + + # set init values + prng_key = prng_key if prng_key is not None else jax.random.PRNGKey(0) + + if generation_config.pad_token_id is None and generation_config.eos_token_id is not None: + if model_kwargs.get("attention_mask") is None: + logger.warning( + "The attention mask and the pad token id were not set. As a consequence, you may observe " + "unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results." + ) + eos_token_id = generation_config.eos_token_id + if isinstance(eos_token_id, list): + eos_token_id = eos_token_id[0] + generation_config.pad_token_id = eos_token_id + + if generation_config.decoder_start_token_id is None and self.config.is_encoder_decoder: + raise ValueError("`decoder_start_token_id` has to be defined for encoder-decoder generation.") + + # decoder-only models should use left-padding for generation (can't be checked with `trace=True`) + if not self.config.is_encoder_decoder and not trace: + if ( + generation_config.pad_token_id is not None + and jnp.sum(input_ids[:, -1] == generation_config.pad_token_id) > 0 + ): + logger.warning( + "A decoder-only architecture is being used, but right-padding was detected! For correct " + "generation results, please set `padding_side='left'` when initializing the tokenizer." + ) + + batch_size = input_ids.shape[0] + + if self.config.is_encoder_decoder: + # add encoder_outputs to model_kwargs + if model_kwargs.get("encoder_outputs") is None: + model_kwargs = self._prepare_encoder_decoder_kwargs_for_generation(input_ids, params, model_kwargs) + # prepare decoder_input_ids for generation + input_ids = self._prepare_decoder_input_ids_for_generation( + batch_size, + decoder_start_token_id=generation_config.decoder_start_token_id, + bos_token_id=generation_config.bos_token_id, + model_kwargs=model_kwargs, + ) + + # Prepare `max_length` depending on other stopping criteria. + input_ids_seq_length = input_ids.shape[-1] + has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None + if has_default_max_length and generation_config.max_new_tokens is None and generation_config.max_length == 20: + # 20 is the default max_length of the generation config + warnings.warn( + f"Using the model-agnostic default `max_length` (={generation_config.max_length}) " + "to control the generation length. recommend setting `max_new_tokens` to control the maximum length of the generation.", + UserWarning, + ) + elif generation_config.max_new_tokens is not None: + if not has_default_max_length and generation_config.max_length is not None: + logger.warning( + f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(=" + f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. " + "Please refer to the documentation for more information. " + "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)" + ) + generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length + else: # by default let's always generate 20 new tokens + if generation_config.max_length == GenerationConfig().max_length: + generation_config.max_length = generation_config.max_length + input_ids_seq_length + max_position_embeddings = getattr(self.config, "max_position_embeddings", None) + if max_position_embeddings is not None: + generation_config.max_length = min(generation_config.max_length, max_position_embeddings) + + if generation_config.min_length is not None and generation_config.min_length > generation_config.max_length: + raise ValueError( + f"Unfeasable length constraints: the minimum length ({generation_config.min_length}) is larger than" + f" the maximum length ({generation_config.max_length})" + ) + if input_ids_seq_length >= generation_config.max_length: + input_ids_string = "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids" + logger.warning( + f"Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to" + f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider" + " increasing`max_new_tokens`." + ) + + logits_processor = self._get_logits_processor( + generation_config=generation_config, + input_ids_seq_length=input_ids_seq_length, + logits_processor=logits_processor, + ) + + if not generation_config.do_sample and generation_config.num_beams == 1: + return self._greedy_search( + input_ids, + generation_config.max_length, + generation_config.pad_token_id, + generation_config.eos_token_id, + logits_processor=logits_processor, + trace=trace, + params=params, + model_kwargs=model_kwargs, + ) + elif generation_config.do_sample and generation_config.num_beams == 1: + logits_warper = self._get_logits_warper(generation_config=generation_config) + return self._sample( + input_ids, + generation_config.max_length, + generation_config.pad_token_id, + generation_config.eos_token_id, + prng_key, + logits_warper=logits_warper, + logits_processor=logits_processor, + trace=trace, + params=params, + model_kwargs=model_kwargs, + ) + elif not generation_config.do_sample and generation_config.num_beams > 1: + # broadcast input_ids & encoder_outputs + input_ids = self._expand_to_num_beams(input_ids, num_beams=generation_config.num_beams) + + if "encoder_outputs" in model_kwargs: + model_kwargs["encoder_outputs"]["last_hidden_state"] = self._expand_to_num_beams( + model_kwargs["encoder_outputs"]["last_hidden_state"], num_beams=generation_config.num_beams + ) + + for kwarg in ["attention_mask", "decoder_attention_mask"]: + if kwarg in model_kwargs: + model_kwargs[kwarg] = self._expand_to_num_beams( + model_kwargs[kwarg], num_beams=generation_config.num_beams + ) + + return self._beam_search( + input_ids, + generation_config.max_length, + generation_config.pad_token_id, + generation_config.eos_token_id, + length_penalty=generation_config.length_penalty, + early_stopping=generation_config.early_stopping, + logits_processor=logits_processor, + trace=trace, + params=params, + num_return_sequences=generation_config.num_return_sequences, + model_kwargs=model_kwargs, + ) + else: + raise NotImplementedError("`Beam sampling is currently not implemented.") + + def _get_logits_warper(self, generation_config: GenerationConfig) -> FlaxLogitsProcessorList: + """ + This class returns a [`FlaxLogitsProcessorList`] list object that contains all relevant [`FlaxLogitsWarper`] + instances used for multinomial sampling. + """ + warpers = FlaxLogitsProcessorList() + + if generation_config.temperature is not None and generation_config.temperature != 1.0: + warpers.append(FlaxTemperatureLogitsWarper(generation_config.temperature)) + if generation_config.top_k is not None and generation_config.top_k != 0: + warpers.append(FlaxTopKLogitsWarper(top_k=generation_config.top_k, min_tokens_to_keep=1)) + if generation_config.top_p is not None and generation_config.top_p < 1.0: + warpers.append(FlaxTopPLogitsWarper(top_p=generation_config.top_p, min_tokens_to_keep=1)) + + return warpers + + def _get_logits_processor( + self, + generation_config: GenerationConfig, + input_ids_seq_length: int, + logits_processor: Optional[FlaxLogitsProcessorList], + ) -> FlaxLogitsProcessorList: + """ + This class returns a [`FlaxLogitsProcessorList`] list object that contains all relevant [`FlaxLogitsProcessor`] + instances used to modify the scores of the language model head. + """ + processors = FlaxLogitsProcessorList() + + if ( + generation_config.min_length is not None + and generation_config.eos_token_id is not None + and generation_config.min_length > -1 + ): + processors.append( + FlaxMinLengthLogitsProcessor(generation_config.min_length, generation_config.eos_token_id) + ) + if generation_config.forced_bos_token_id is not None: + processors.append(FlaxForcedBOSTokenLogitsProcessor(generation_config.forced_bos_token_id)) + if generation_config.forced_eos_token_id is not None: + processors.append( + FlaxForcedEOSTokenLogitsProcessor(generation_config.max_length, generation_config.forced_eos_token_id) + ) + if generation_config.suppress_tokens is not None: + processors.append(FlaxSuppressTokensLogitsProcessor(generation_config.suppress_tokens)) + if generation_config.begin_suppress_tokens is not None: + begin_index = input_ids_seq_length + begin_index = ( + begin_index + if (input_ids_seq_length > 1 or generation_config.forced_bos_token_id is None) + else begin_index + 1 + ) + if generation_config.forced_decoder_ids is not None and len(generation_config.forced_decoder_ids) > 0: + # generation starts after the last token that is forced + begin_index += generation_config.forced_decoder_ids[-1][0] + processors.append( + FlaxSuppressTokensAtBeginLogitsProcessor(generation_config.begin_suppress_tokens, begin_index) + ) + if generation_config.forced_decoder_ids is not None: + forced_decoder_ids = [ + [input_ids_seq_length + i[0] - 1, i[1]] for i in generation_config.forced_decoder_ids + ] + processors.append(FlaxForceTokensLogitsProcessor(forced_decoder_ids)) + if generation_config.no_repeat_ngram_size is not None and generation_config.no_repeat_ngram_size > 0: + processors.append(FlaxNoRepeatNGramLogitsProcessor(generation_config.no_repeat_ngram_size)) + processors = self._merge_criteria_processor_list(processors, logits_processor) + + return processors + + def _merge_criteria_processor_list( + self, + default_list: FlaxLogitsProcessorList, + custom_list: FlaxLogitsProcessorList, + ) -> FlaxLogitsProcessorList: + if len(custom_list) == 0: + return default_list + for default in default_list: + for custom in custom_list: + if type(custom) is type(default): + object_type = "logits processor" + raise ValueError( + f"A custom {object_type} of type {type(custom)} with values {custom} has been passed to" + f" `generate`, but it has already been created with the values {default}. {default} has been" + " created by passing the corresponding arguments to generate or by the model's config default" + f" values. If you just want to change the default values of {object_type} consider passing" + f" them as arguments to `generate` instead of using a custom {object_type}." + ) + default_list.extend(custom_list) + return default_list + + def _greedy_search( + self, + input_ids: None, + max_length: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + logits_processor: Optional[FlaxLogitsProcessorList] = None, + trace: bool = True, + params: Optional[Dict[str, jnp.ndarray]] = None, + model_kwargs: Optional[Dict[str, jnp.ndarray]] = None, + ): + # init values + max_length = max_length if max_length is not None else self.generation_config.max_length + pad_token_id = pad_token_id if pad_token_id is not None else self.generation_config.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.generation_config.eos_token_id + + batch_size, cur_len = input_ids.shape + + eos_token_id = jnp.array(eos_token_id, dtype=jnp.int32 if eos_token_id is not None else None) + pad_token_id = jnp.array(pad_token_id, dtype=jnp.int32) + cur_len = jnp.array(cur_len) + + # per batch-item holding current token in loop. + sequences = jnp.full((batch_size, max_length), pad_token_id, dtype=jnp.int32) + sequences = lax.dynamic_update_slice(sequences, input_ids, (0, 0)) + + # per batch-item state bit indicating if sentence has finished. + is_sent_finished = jnp.zeros((batch_size,), dtype=jnp.bool_) + + # For Seq2Seq generation, we only need to use the decoder instead of the whole model in generation loop + # and pass it the `encoder_outputs`, which are part of the `model_kwargs`. + model = self.decode if self.config.is_encoder_decoder else self + # initialize model specific kwargs + model_kwargs = self.prepare_inputs_for_generation(input_ids, max_length, **model_kwargs) + + # initialize state + state = GreedyState( + cur_len=cur_len, + sequences=sequences, + running_token=input_ids, + is_sent_finished=is_sent_finished, + model_kwargs=model_kwargs, + ) + + def greedy_search_cond_fn(state): + """state termination condition fn.""" + has_reached_max_length = state.cur_len == max_length + all_sequence_finished = jnp.all(state.is_sent_finished) + finish_generation = jnp.logical_or(has_reached_max_length, all_sequence_finished) + return ~finish_generation + + def greedy_search_body_fn(state): + """state update fn.""" + model_outputs = model(state.running_token, params=params, **state.model_kwargs) + logits = model_outputs.logits[:, -1] + + # apply min_length, ... + logits = logits_processor(state.sequences, logits, state.cur_len) + + next_token = jnp.argmax(logits, axis=-1) + + next_token = next_token * ~state.is_sent_finished + pad_token_id * state.is_sent_finished + next_is_sent_finished = state.is_sent_finished | (next_token == eos_token_id) + next_token = next_token[:, None] + + next_sequences = lax.dynamic_update_slice(state.sequences, next_token, (0, state.cur_len)) + next_model_kwargs = self.update_inputs_for_generation(model_outputs, state.model_kwargs) + return GreedyState( + cur_len=state.cur_len + 1, + sequences=next_sequences, + running_token=next_token, + is_sent_finished=next_is_sent_finished, + model_kwargs=next_model_kwargs, + ) + + # The very first prompt often has sequence length > 1, so run outside of `lax.while_loop` to comply with TPU + if input_ids.shape[1] > 1: + state = greedy_search_body_fn(state) + + if not trace: + state = self._run_loop_in_debug(greedy_search_cond_fn, greedy_search_body_fn, state) + else: + state = lax.while_loop(greedy_search_cond_fn, greedy_search_body_fn, state) + + return FlaxGreedySearchOutput(sequences=state.sequences) + + def _sample( + self, + input_ids: None, + max_length: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + prng_key: Optional[jnp.ndarray] = None, + logits_processor: Optional[FlaxLogitsProcessorList] = None, + logits_warper: Optional[FlaxLogitsProcessorList] = None, + trace: bool = True, + params: Optional[Dict[str, jnp.ndarray]] = None, + model_kwargs: Optional[Dict[str, jnp.ndarray]] = None, + ): + # init values + max_length = max_length if max_length is not None else self.generation_config.max_length + pad_token_id = pad_token_id if pad_token_id is not None else self.generation_config.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.generation_config.eos_token_id + prng_key = prng_key if prng_key is not None else jax.random.PRNGKey(0) + + batch_size, cur_len = input_ids.shape + + eos_token_id = jnp.array(eos_token_id, dtype=jnp.int32 if eos_token_id is not None else None) + pad_token_id = jnp.array(pad_token_id, dtype=jnp.int32) + cur_len = jnp.array(cur_len) + + # per batch-item holding current token in loop. + sequences = jnp.full((batch_size, max_length), pad_token_id, dtype=jnp.int32) + sequences = lax.dynamic_update_slice(sequences, input_ids, (0, 0)) + + # per batch-item state bit indicating if sentence has finished. + is_sent_finished = jnp.zeros((batch_size,), dtype=jnp.bool_) + + # For Seq2Seq generation, we only need to use the decoder instead of the whole model in generation loop + # and pass it the `encoder_outputs`, which are part of the `model_kwargs`. + model = self.decode if self.config.is_encoder_decoder else self + + # initialize model specific kwargs + model_kwargs = self.prepare_inputs_for_generation(input_ids, max_length, **model_kwargs) + + # initialize state + state = SampleState( + cur_len=cur_len, + sequences=sequences, + running_token=input_ids, + is_sent_finished=is_sent_finished, + prng_key=prng_key, + model_kwargs=model_kwargs, + ) + + def sample_search_cond_fn(state): + """state termination condition fn.""" + has_reached_max_length = state.cur_len == max_length + all_sequence_finished = jnp.all(state.is_sent_finished) + finish_generation = jnp.logical_or(has_reached_max_length, all_sequence_finished) + return ~finish_generation + + def sample_search_body_fn(state): + """state update fn.""" + prng_key, prng_key_next = jax.random.split(state.prng_key) + model_outputs = model(state.running_token, params=params, **state.model_kwargs) + + logits = model_outputs.logits[:, -1] + + # apply min_length, ... + logits = logits_processor(state.sequences, logits, state.cur_len) + # apply top_p, top_k, temperature + logits = logits_warper(logits, logits, state.cur_len) + + next_token = jax.random.categorical(prng_key, logits, axis=-1) + + next_token = next_token * ~state.is_sent_finished + pad_token_id * state.is_sent_finished + next_is_sent_finished = state.is_sent_finished | (next_token == eos_token_id) + next_token = next_token[:, None] + + next_sequences = lax.dynamic_update_slice(state.sequences, next_token, (0, state.cur_len)) + next_model_kwargs = self.update_inputs_for_generation(model_outputs, state.model_kwargs) + + return SampleState( + cur_len=state.cur_len + 1, + sequences=next_sequences, + running_token=next_token, + is_sent_finished=next_is_sent_finished, + model_kwargs=next_model_kwargs, + prng_key=prng_key_next, + ) + + # The very first prompt often has sequence length > 1, so run outside of `lax.while_loop` to comply with TPU + if input_ids.shape[1] > 1: + state = sample_search_body_fn(state) + + if not trace: + state = self._run_loop_in_debug(sample_search_cond_fn, sample_search_body_fn, state) + else: + state = lax.while_loop(sample_search_cond_fn, sample_search_body_fn, state) + + return FlaxSampleOutput(sequences=state.sequences) + + def _beam_search( + self, + input_ids: None, + max_length: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + length_penalty: Optional[float] = None, + early_stopping: Optional[Union[bool, str]] = None, + logits_processor: Optional[FlaxLogitsProcessorList] = None, + trace: bool = True, + params: Optional[Dict[str, jnp.ndarray]] = None, + num_return_sequences: Optional[int] = None, + model_kwargs: Optional[Dict[str, jnp.ndarray]] = None, + ): + """ + This beam search function is heavily inspired by Flax's official example: + https://github.com/google/flax/blob/main/examples/wmt/decode.py + """ + + def flatten_beam_dim(tensor): + """Flattens the first two dimensions of a non-scalar array.""" + # ignore scalars (e.g. cache index) + if tensor.ndim == 0: + return tensor + return tensor.reshape((tensor.shape[0] * tensor.shape[1],) + tensor.shape[2:]) + + def unflatten_beam_dim(tensor, batch_size, num_beams): + """Unflattens the first, flat batch*beam dimension of a non-scalar array.""" + # ignore scalars (e.g. cache index) + if tensor.ndim == 0: + return tensor + return tensor.reshape((batch_size, num_beams) + tensor.shape[1:]) + + def gather_beams(nested, beam_indices, batch_size, new_num_beams): + """ + Gathers the beam slices indexed by beam_indices into new beam array. + """ + batch_indices = jnp.reshape( + jnp.arange(batch_size * new_num_beams) // new_num_beams, (batch_size, new_num_beams) + ) + + def gather_fn(tensor): + # ignore scalars (e.g. cache index) + if tensor.ndim == 0: + return tensor + else: + return tensor[batch_indices, beam_indices] + + return jax.tree_util.tree_map(gather_fn, nested) + + # init values + max_length = max_length if max_length is not None else self.generation_config.max_length + pad_token_id = pad_token_id if pad_token_id is not None else self.generation_config.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.generation_config.eos_token_id + length_penalty = length_penalty if length_penalty is not None else self.generation_config.length_penalty + early_stopping = early_stopping if early_stopping is not None else self.generation_config.early_stopping + num_return_sequences = ( + num_return_sequences if num_return_sequences is not None else self.generation_config.num_return_sequences + ) + + batch_size, num_beams, cur_len = input_ids.shape + + eos_token_id = jnp.array(eos_token_id, dtype=jnp.int32 if eos_token_id is not None else None) + pad_token_id = jnp.array(pad_token_id, dtype=jnp.int32) + cur_len = jnp.array(cur_len) + + # record the prompt length of decoder + decoder_prompt_len = input_ids.shape[-1] + + # per batch,beam-item holding current token in loop. + sequences = jnp.full((batch_size, num_beams, max_length), pad_token_id, dtype=jnp.int32) + running_sequences = jnp.full((batch_size, num_beams, max_length), pad_token_id, dtype=jnp.int32) + running_sequences = lax.dynamic_update_slice(sequences, input_ids, (0, 0, 0)) + + # per batch,beam-item state bit indicating if sentence has finished. + is_sent_finished = jnp.zeros((batch_size, num_beams), dtype=jnp.bool_) + + # per batch,beam-item score, logprobs + running_scores = jnp.tile(jnp.array([0.0] + [np.array(-1.0e7)] * (num_beams - 1)), [batch_size, 1]) + scores = jnp.ones((batch_size, num_beams)) * np.array(-1.0e7) + + # For Seq2Seq generation, we only need to use the decoder instead of the whole model in generation loop + # and pass it the `encoder_outputs`, which are part of the `model_kwargs`. + model = self.decode if self.config.is_encoder_decoder else self + + # flatten beam dim + if "encoder_outputs" in model_kwargs: + model_kwargs["encoder_outputs"]["last_hidden_state"] = flatten_beam_dim( + model_kwargs["encoder_outputs"]["last_hidden_state"] + ) + for kwarg in ["attention_mask", "decoder_attention_mask"]: + if kwarg in model_kwargs: + model_kwargs[kwarg] = flatten_beam_dim(model_kwargs[kwarg]) + + # initialize model specific kwargs + model_kwargs = self.prepare_inputs_for_generation(flatten_beam_dim(input_ids), max_length, **model_kwargs) + + # initialize state + state = BeamSearchState( + cur_len=cur_len, + running_sequences=running_sequences, + running_scores=running_scores, + sequences=sequences, + scores=scores, + is_sent_finished=is_sent_finished, + model_kwargs=model_kwargs, + ) + + def beam_search_cond_fn(state): + """beam search state termination condition fn.""" + + # 1. is less than max length? + not_max_length_yet = state.cur_len < max_length + + # 2. can the new beams still improve? + # early_stopping == False -> apply heuristic = always get the best score from `cur_len`. See the discussion + # below for more details. + # https://github.com/huggingface/transformers/pull/20901#issuecomment-1369845565 + # early_stopping == "never" -> compute the best score from max_length or cur_len, depending on the sign of + # length_penalty. Positive length_penalty favors longer sequences, thus we use max_length there. + if early_stopping == "never" and length_penalty > 0.0: + best_running_score = state.running_scores[:, :1] / ( + (max_length - decoder_prompt_len) ** length_penalty + ) + else: + best_running_score = state.running_scores[:, :1] / ( + (state.cur_len - decoder_prompt_len) ** length_penalty + ) + worst_finished_score = jnp.where( + state.is_sent_finished, jnp.min(state.scores, axis=1, keepdims=True), np.array(-1.0e7) + ) + improvement_still_possible = jnp.any(best_running_score > worst_finished_score) + + # 3. is there still a beam that has not finished? + still_open_beam = ~(jnp.all(state.is_sent_finished) & (early_stopping is True)) + + return not_max_length_yet & still_open_beam & improvement_still_possible + + def beam_search_body_fn(state, input_ids_length=1): + """beam search state update fn.""" + # 1. Forward current tokens + # Collect the current position slice along length to feed the fast + # autoregressive decoder model. Flatten the beam dimension into batch + # dimension for feeding into the model. + # unflatten beam dimension + # Unflatten beam dimension in attention cache arrays + input_token = flatten_beam_dim( + lax.dynamic_slice( + state.running_sequences, + (0, 0, state.cur_len - input_ids_length), + (batch_size, num_beams, input_ids_length), + ) + ) + model_outputs = model(input_token, params=params, **state.model_kwargs) + + logits = unflatten_beam_dim(model_outputs.logits[:, -1], batch_size, num_beams) + cache = jax.tree_util.tree_map( + lambda tensor: unflatten_beam_dim(tensor, batch_size, num_beams), model_outputs.past_key_values + ) + + # adapt logits for FlaxMarianMTModel + logits = self._adapt_logits_for_beam_search(logits) + + # 2. Compute log probs + # get log probabilities from logits, + # process logits with processors (*e.g.* min_length, ...), and + # add new logprobs to existing running logprobs scores. + log_probs = jax.nn.log_softmax(logits) + log_probs = logits_processor( + flatten_beam_dim(state.running_sequences), flatten_beam_dim(log_probs), state.cur_len + ) + log_probs = unflatten_beam_dim(log_probs, batch_size, num_beams) + log_probs = log_probs + jnp.expand_dims(state.running_scores, axis=2) + vocab_size = log_probs.shape[2] + log_probs = log_probs.reshape((batch_size, num_beams * vocab_size)) + + # 3. Retrieve top-K + # Each item in batch has num_beams * vocab_size candidate sequences. + # For each item, get the top 2*k candidates with the highest log- + # probabilities. We gather the top 2*K beams here so that even if the best + # K sequences reach EOS simultaneously, we have another K sequences + # remaining to continue the live beam search. + # Gather the top 2*K scores from _all_ beams. + # Gather 2*k top beams. + # Recover the beam index by floor division. + # Recover token id by modulo division and expand Id array for broadcasting. + # Update sequences for the 2*K top-k new sequences. + beams_to_keep = 2 * num_beams + topk_log_probs, topk_indices = lax.top_k(log_probs, k=beams_to_keep) + topk_beam_indices = topk_indices // vocab_size + topk_running_sequences = gather_beams( + state.running_sequences, topk_beam_indices, batch_size, beams_to_keep + ) + topk_ids = jnp.expand_dims(topk_indices % vocab_size, axis=2) + topk_sequences = lax.dynamic_update_slice(topk_running_sequences, topk_ids, (0, 0, state.cur_len)) + + # 4. Check which sequences have ended + # Update current sequences: + # Did any of these sequences reach an end marker? + # To prevent these just finished sequences from being added to the current sequences + # set of active beam search sequences, set their log probs to a very large + # negative value. + did_topk_just_finished = topk_sequences[:, :, state.cur_len] == eos_token_id + running_topk_log_probs = topk_log_probs + did_topk_just_finished * np.array(-1.0e7) + # 5. Get running sequences scores for next + # Determine the top k beam indices (from top 2*k beams) from log probs + # and gather top k beams (from top 2*k beams). + next_topk_indices = lax.top_k(running_topk_log_probs, k=num_beams)[1] + next_running_sequences, next_running_scores = gather_beams( + [topk_sequences, running_topk_log_probs], next_topk_indices, batch_size, num_beams + ) + + # 6. Process topk logits + # Further process log probs: + # - add length penalty + # - make sure no scores can be added anymore if beam is full + # - make sure still running sequences cannot be chosen as finalized beam + topk_log_probs = topk_log_probs / ((state.cur_len + 1 - decoder_prompt_len) ** length_penalty) + beams_in_batch_are_full = jnp.broadcast_to( + state.is_sent_finished.all(axis=-1, keepdims=True), did_topk_just_finished.shape + ) & (early_stopping is True) + add_penalty = ~did_topk_just_finished | beams_in_batch_are_full + topk_log_probs += add_penalty * np.array(-1.0e7) + + # 7. Get scores, sequences, is sentence finished for next. + # Combine sequences, scores, and flags along the beam dimension and compare + # new finished sequence scores to existing finished scores and select the + # best from the new set of beams + merged_sequences = jnp.concatenate([state.sequences, topk_sequences], axis=1) + merged_scores = jnp.concatenate([state.scores, topk_log_probs], axis=1) + merged_is_sent_finished = jnp.concatenate([state.is_sent_finished, did_topk_just_finished], axis=1) + topk_merged_indices = lax.top_k(merged_scores, k=num_beams)[1] + next_sequences, next_scores, next_is_sent_finished = gather_beams( + [merged_sequences, merged_scores, merged_is_sent_finished], topk_merged_indices, batch_size, num_beams + ) + + # 8. Update model kwargs. + # Determine the top k beam indices from the original set of all beams. + # With these, gather the top k beam-associated caches. + next_running_indices = gather_beams(topk_beam_indices, next_topk_indices, batch_size, num_beams) + next_cache = gather_beams(cache, next_running_indices, batch_size, num_beams) + model_outputs["past_key_values"] = jax.tree_util.tree_map(lambda x: flatten_beam_dim(x), next_cache) + next_model_kwargs = self.update_inputs_for_generation(model_outputs, state.model_kwargs) + + return BeamSearchState( + cur_len=state.cur_len + 1, + running_scores=next_running_scores, + running_sequences=next_running_sequences, + scores=next_scores, + sequences=next_sequences, + is_sent_finished=next_is_sent_finished, + model_kwargs=next_model_kwargs, + ) + + # Always run first iteration outside of `lax.while_loop` to avoid calling `beam_search_cond_fn` + # when `state.cur_len` equals `decoder_prompt_len`. This also helps to comply with TPU when + # the very first prompt has sequence length > 1. + state = partial(beam_search_body_fn, input_ids_length=input_ids.shape[-1])(state) + + if not trace: + state = self._run_loop_in_debug(beam_search_cond_fn, beam_search_body_fn, state) + else: + state = lax.while_loop(beam_search_cond_fn, beam_search_body_fn, state) + + # Account for the edge-case where there are no finished sequences for a + # particular batch item. If so, return running sequences for that batch item. + none_finished = jnp.any(state.is_sent_finished, axis=1) + sequences = jnp.where(none_finished[:, None, None], state.sequences, state.running_sequences) + scores = jnp.where(none_finished[:, None], state.scores, state.running_scores) + + # Take best beams for each batch (the score is sorted in descending order) + sequences = flatten_beam_dim(sequences[:, :num_return_sequences, :]) + scores = flatten_beam_dim(scores[:, :num_return_sequences]) + + return FlaxBeamSearchOutput(sequences=sequences, scores=scores) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flex_attention.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flex_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..b4d3b6fc3e01b37ec53bd50f1fc03e40670d64f6 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/flex_attention.py @@ -0,0 +1,255 @@ +""" +Partially inspired by torchtune's flex attention implementation + +Citation: +@software{torchtune, + title = {torchtune: PyTorch's finetuning library}, + author = {torchtune maintainers and contributors}, + url = {https//github.com/pytorch/torchtune}, + license = {BSD-3-Clause}, + month = apr, + year = {2024} +} +""" +# coding=utf-8 +# Copyright 2025 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple, Union + +import torch +from packaging import version + +from ..utils import is_torch_flex_attn_available +from ..utils.import_utils import _torch_version + + +if is_torch_flex_attn_available(): + from torch.nn.attention.flex_attention import BlockMask, flex_attention + from torch.nn.attention.flex_attention import ( + create_block_mask as create_block_causal_mask_flex, + ) + + +class WrappedFlexAttention: + """ + We are doing a singleton class so that flex attention is compiled once when it's first called. + """ + + _instance = None + _is_flex_compiled = False + _compiled_flex_attention = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + # Create a new instance if one doesn't already exist + cls._instance = super().__new__(cls) + return cls._instance + + @torch.compiler.disable(recursive=False) + def __init__(self, training): + """ + Initialize or update the singleton instance. + """ + if not self._is_flex_compiled or training != self.training: + # In PyTorch 2.6.0, there's a known issue with flex attention compilation which may + # cause errors. The suggested fix is to compile with "max-autotune-no-cudagraphs" + # see https://github.com/pytorch/pytorch/issues/146260 for training + self.training = training + if version.parse(_torch_version).base_version == "2.6.0" and training: + self._compiled_flex_attention = torch.compile( + flex_attention, dynamic=False, mode="max-autotune-no-cudagraphs" + ) + else: + self._compiled_flex_attention = torch.compile(flex_attention) + self._is_flex_compiled = True + + def __call__(self): + return self._compiled_flex_attention + + +Offset = Union[torch.Tensor, int] + + +def make_flex_block_causal_mask( + attention_mask_2d: torch.Tensor, + attention_chunk_size: Optional[int] = None, + query_length=None, + key_length=None, + offsets: Optional[Tuple[Offset, Offset]] = None, +) -> "BlockMask": + """ + Create a block causal document mask for a batch of sequences, both packed and unpacked. + Create Block causal logic and passing it into :func:`torch.nn.attention.flex_attention.create_block_mask`. + The resultant BlockMask is a compressed representation of the full block causal + mask. BlockMask is essential for performant computation of flex attention. + See: https://pytorch.org/blog/flexattention/ + + Args: + attention_mask_2d (torch.Tensor): Attention mask for packed and padded sequences + of shape (batch_size, total_seq_len). e.g. + + For unpacked sequence: + [[1, 1, 1, 1, 0, 0, 0], + [1, 1, 1, 1, 1, 0, 0]] + + For packed sequence: + [[1, 1, 1, 2, 2, 2, 0], + [1, 1, 2, 2, 2, 3, 3]] + + Returns: + BlockMask + """ + batch_size, total_seq_len = attention_mask_2d.shape + if not key_length: + key_length = total_seq_len + if not query_length: + query_length = total_seq_len + attention_mask_2d = torch.nn.functional.pad(attention_mask_2d, value=0, pad=(0, key_length)) + device = attention_mask_2d.device + document_ids = attention_mask_2d.clone() + + if attention_chunk_size is not None: + # we create an arange, then we just // by chunk size to get [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3] + document_ids = (document_ids.fill_(1).cumsum(-1) - 1) // (attention_chunk_size) + + # Instead of passing a tensor mask, flex attention requires a mask_mod function + # that determines which elements of QK^T should be included in the attention + # computation prior to the softmax. For sample packing, we need both the + # logic for both causal mask and document mask. See PyTorch's official + # blog post for more details: https://pytorch.org/blog/flexattention/#mask-mods + def causal_mask_mod(batch_idx, head_idx, q_idx, kv_idx): + """ + Defines the logic of a block causal mask by combining both a standard causal mask + and a block diagonal document mask. + + See :func:`~torchtune.modules.attention_utils.create_block_causal_mask` + for an illustration. + """ + causal_mask = q_idx >= kv_idx # not valid when decoding + document_mask = document_ids[batch_idx, q_idx] == document_ids[batch_idx, kv_idx] + padding_mask = attention_mask_2d[batch_idx, q_idx] > 0 + final_mask = causal_mask & padding_mask & document_mask + return final_mask + + if offsets is not None: + q_offset = offsets[0] + kv_offset = offsets[1] + + def mask_mod(batch_idx, head_idx, q_idx, kv_idx): + offset_q = q_idx + q_offset + offset_kv = kv_idx + kv_offset + return causal_mask_mod(batch_idx, head_idx, offset_q, offset_kv) + else: + mask_mod = causal_mask_mod + return create_block_causal_mask_flex( + mask_mod=mask_mod, + B=batch_size, + H=None, # attention head + Q_LEN=query_length, + KV_LEN=key_length, + device=device, + _compile=True, + ) + + +@torch.compiler.disable(recursive=False) +def compile_friendly_flex_attention( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + training=False, + **kwargs, +) -> torch.Tensor: + # First call initialise singleton wrapper object, second call invokes the object method to return compiled flex attention + flex_attention_compiled = WrappedFlexAttention(training)() + return flex_attention_compiled( + query, + key, + value, + **kwargs, + ) + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def flex_attention_forward( + module: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Union[torch.Tensor, "BlockMask"], + scaling: Optional[float] = None, + softcap: Optional[float] = None, + head_mask: Optional[torch.Tensor] = None, + **kwargs, +) -> Tuple[torch.Tensor, torch.Tensor]: + block_mask = None + causal_mask = None + if isinstance(attention_mask, BlockMask): + block_mask = attention_mask + else: + causal_mask = attention_mask + + if causal_mask is not None: + causal_mask = causal_mask[:, :, :, : key.shape[-2]] + + def score_mod(score, batch_idx, head_idx, q_idx, kv_idx): + if softcap is not None: + score = softcap * torch.tanh(score / softcap) + if causal_mask is not None: + score = score + causal_mask[batch_idx][0][q_idx][kv_idx] + if head_mask is not None: + score = score + head_mask[batch_idx][head_idx][0][0] + return score + + enable_gqa = True + num_local_query_heads = query.shape[1] + + # When running TP this helps: + if not ((num_local_query_heads & (num_local_query_heads - 1)) == 0): + key = repeat_kv(key, query.shape[1] // key.shape[1]) + value = repeat_kv(value, query.shape[1] // value.shape[1]) + enable_gqa = False + + kernel_options = kwargs.get("kernel_options", None) + attn_output, attention_weights = compile_friendly_flex_attention( + query, + key, + value, + score_mod=score_mod, + block_mask=block_mask, + enable_gqa=enable_gqa, + scale=scaling, + kernel_options=kernel_options, + # Last time checked on PyTorch == 2.5.1: Flex Attention always computes the lse regardless. + # For simplification, we thus always return it as no additional computations are introduced. + return_lse=True, + training=module.training, + ) + # lse is returned in float32 + attention_weights = attention_weights.to(value.dtype) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attention_weights diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/fsdp.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/fsdp.py new file mode 100644 index 0000000000000000000000000000000000000000..332231c04f433f4ebd1364347c5cbce62dd498e6 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/fsdp.py @@ -0,0 +1,38 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..utils import is_torch_available + + +if TYPE_CHECKING: + from torch import nn + + +def is_fsdp_managed_module(module: nn.Module) -> bool: + if not is_torch_available(): + return False + + import torch + + if not torch.distributed.is_available(): + return False + + import torch.distributed.fsdp + + return isinstance(module, torch.distributed.fsdp.FullyShardedDataParallel) or getattr( + module, "_is_fsdp_managed_module", False + ) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/generation_config.json b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..205dc2c5dc17bbd8798ff68735649b13420f50ec --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 151643, + "eos_token_id": 151645, + "transformers_version": "4.51.3", + "use_cache": false +} diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/generic.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/generic.py new file mode 100644 index 0000000000000000000000000000000000000000..65a3efaed5ab5cfaf0935638e93c20c23aecef8d --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/generic.py @@ -0,0 +1,975 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Generic utilities +""" + +import inspect +import json +import os +import tempfile +import warnings +from collections import OrderedDict, UserDict +from collections.abc import Iterable, MutableMapping +from contextlib import ExitStack, contextmanager +from dataclasses import fields, is_dataclass +from enum import Enum +from functools import partial, wraps +from typing import Any, ContextManager, Optional, TypedDict + +import numpy as np +from packaging import version + +from .import_utils import ( + get_torch_version, + is_flax_available, + is_mlx_available, + is_tf_available, + is_torch_available, + is_torch_fx_proxy, +) + + +if is_torch_available(): + # required for @can_return_tuple decorator to work with torchdynamo + import torch # noqa: F401 + + +class cached_property(property): + """ + Descriptor that mimics @property but caches output in member variable. + + From tensorflow_datasets + + Built-in in functools from Python 3.8. + """ + + def __get__(self, obj, objtype=None): + # See docs.python.org/3/howto/descriptor.html#properties + if obj is None: + return self + if self.fget is None: + raise AttributeError("unreadable attribute") + attr = "__cached_" + self.fget.__name__ + cached = getattr(obj, attr, None) + if cached is None: + cached = self.fget(obj) + setattr(obj, attr, cached) + return cached + + +# vendored from distutils.util +def strtobool(val): + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are 'n', 'no', 'f', 'false', 'off', and '0'. + Raises ValueError if 'val' is anything else. + """ + val = val.lower() + if val in {"y", "yes", "t", "true", "on", "1"}: + return 1 + if val in {"n", "no", "f", "false", "off", "0"}: + return 0 + raise ValueError(f"invalid truth value {val!r}") + + +def infer_framework_from_repr(x): + """ + Tries to guess the framework of an object `x` from its repr (brittle but will help in `is_tensor` to try the + frameworks in a smart order, without the need to import the frameworks). + """ + representation = str(type(x)) + if representation.startswith(" + + You can't unpack a `ModelOutput` directly. Use the [`~utils.ModelOutput.to_tuple`] method to convert it to a tuple + before. + + + """ + + def __init_subclass__(cls) -> None: + """Register subclasses as pytree nodes. + + This is necessary to synchronize gradients when using `torch.nn.parallel.DistributedDataParallel` with + `static_graph=True` with modules that output `ModelOutput` subclasses. + """ + if is_torch_available(): + if version.parse(get_torch_version()) >= version.parse("2.2"): + _torch_pytree.register_pytree_node( + cls, + _model_output_flatten, + partial(_model_output_unflatten, output_type=cls), + serialized_type_name=f"{cls.__module__}.{cls.__name__}", + ) + else: + _torch_pytree._register_pytree_node( + cls, + _model_output_flatten, + partial(_model_output_unflatten, output_type=cls), + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Subclasses of ModelOutput must use the @dataclass decorator + # This check is done in __init__ because the @dataclass decorator operates after __init_subclass__ + # issubclass() would return True for issubclass(ModelOutput, ModelOutput) when False is needed + # Just need to check that the current class is not ModelOutput + is_modeloutput_subclass = self.__class__ != ModelOutput + + if is_modeloutput_subclass and not is_dataclass(self): + raise TypeError( + f"{self.__module__}.{self.__class__.__name__} is not a dataclass." + " This is a subclass of ModelOutput and so must use the @dataclass decorator." + ) + + def __post_init__(self): + """Check the ModelOutput dataclass. + + Only occurs if @dataclass decorator has been used. + """ + class_fields = fields(self) + + # Safety and consistency checks + if not len(class_fields): + raise ValueError(f"{self.__class__.__name__} has no fields.") + if not all(field.default is None for field in class_fields[1:]): + raise ValueError(f"{self.__class__.__name__} should not have more than one required field.") + + first_field = getattr(self, class_fields[0].name) + other_fields_are_none = all(getattr(self, field.name) is None for field in class_fields[1:]) + + if other_fields_are_none and not is_tensor(first_field): + if isinstance(first_field, dict): + iterator = first_field.items() + first_field_iterator = True + else: + try: + iterator = iter(first_field) + first_field_iterator = True + except TypeError: + first_field_iterator = False + + # if we provided an iterator as first field and the iterator is a (key, value) iterator + # set the associated fields + if first_field_iterator: + for idx, element in enumerate(iterator): + if ( + not isinstance(element, (list, tuple)) + or not len(element) == 2 + or not isinstance(element[0], str) + ): + if idx == 0: + # If we do not have an iterator of key/values, set it as attribute + self[class_fields[0].name] = first_field + else: + # If we have a mixed iterator, raise an error + raise ValueError( + f"Cannot set key/value for {element}. It needs to be a tuple (key, value)." + ) + break + setattr(self, element[0], element[1]) + if element[1] is not None: + self[element[0]] = element[1] + elif first_field is not None: + self[class_fields[0].name] = first_field + else: + for field in class_fields: + v = getattr(self, field.name) + if v is not None: + self[field.name] = v + + def __delitem__(self, *args, **kwargs): + raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + + def setdefault(self, *args, **kwargs): + raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + + def pop(self, *args, **kwargs): + raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + + def update(self, *args, **kwargs): + raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + + def __getitem__(self, k): + if isinstance(k, str): + inner_dict = dict(self.items()) + return inner_dict[k] + else: + return self.to_tuple()[k] + + def __setattr__(self, name, value): + if name in self.keys() and value is not None: + # Don't call self.__setitem__ to avoid recursion errors + super().__setitem__(name, value) + super().__setattr__(name, value) + + def __setitem__(self, key, value): + # Will raise a KeyException if needed + super().__setitem__(key, value) + # Don't call self.__setattr__ to avoid recursion errors + super().__setattr__(key, value) + + def __reduce__(self): + if not is_dataclass(self): + return super().__reduce__() + callable, _args, *remaining = super().__reduce__() + args = tuple(getattr(self, field.name) for field in fields(self)) + return callable, args, *remaining + + def to_tuple(self) -> tuple[Any]: + """ + Convert self to a tuple containing all the attributes/keys that are not `None`. + """ + return tuple(self[k] for k in self.keys()) + + +if is_torch_available(): + import torch.utils._pytree as _torch_pytree + + def _model_output_flatten(output: ModelOutput) -> tuple[list[Any], "_torch_pytree.Context"]: + return list(output.values()), list(output.keys()) + + def _model_output_unflatten( + values: Iterable[Any], + context: "_torch_pytree.Context", + output_type=None, + ) -> ModelOutput: + return output_type(**dict(zip(context, values))) + + if version.parse(get_torch_version()) >= version.parse("2.2"): + _torch_pytree.register_pytree_node( + ModelOutput, + _model_output_flatten, + partial(_model_output_unflatten, output_type=ModelOutput), + serialized_type_name=f"{ModelOutput.__module__}.{ModelOutput.__name__}", + ) + else: + _torch_pytree._register_pytree_node( + ModelOutput, + _model_output_flatten, + partial(_model_output_unflatten, output_type=ModelOutput), + ) + + +class ExplicitEnum(str, Enum): + """ + Enum with more explicit error message for missing values. + """ + + @classmethod + def _missing_(cls, value): + raise ValueError( + f"{value} is not a valid {cls.__name__}, please select one of {list(cls._value2member_map_.keys())}" + ) + + +class PaddingStrategy(ExplicitEnum): + """ + Possible values for the `padding` argument in [`PreTrainedTokenizerBase.__call__`]. Useful for tab-completion in an + IDE. + """ + + LONGEST = "longest" + MAX_LENGTH = "max_length" + DO_NOT_PAD = "do_not_pad" + + +class TensorType(ExplicitEnum): + """ + Possible values for the `return_tensors` argument in [`PreTrainedTokenizerBase.__call__`]. Useful for + tab-completion in an IDE. + """ + + PYTORCH = "pt" + TENSORFLOW = "tf" + NUMPY = "np" + JAX = "jax" + MLX = "mlx" + + +class ContextManagers: + """ + Wrapper for `contextlib.ExitStack` which enters a collection of context managers. Adaptation of `ContextManagers` + in the `fastcore` library. + """ + + def __init__(self, context_managers: list[ContextManager]): + self.context_managers = context_managers + self.stack = ExitStack() + + def __enter__(self): + for context_manager in self.context_managers: + self.stack.enter_context(context_manager) + + def __exit__(self, *args, **kwargs): + self.stack.__exit__(*args, **kwargs) + + +def can_return_loss(model_class): + """ + Check if a given model can return loss. + + Args: + model_class (`type`): The class of the model. + """ + framework = infer_framework(model_class) + if framework == "tf": + signature = inspect.signature(model_class.call) # TensorFlow models + elif framework == "pt": + signature = inspect.signature(model_class.forward) # PyTorch models + else: + signature = inspect.signature(model_class.__call__) # Flax models + + for p in signature.parameters: + if p == "return_loss" and signature.parameters[p].default is True: + return True + + return False + + +def find_labels(model_class): + """ + Find the labels used by a given model. + + Args: + model_class (`type`): The class of the model. + """ + model_name = model_class.__name__ + framework = infer_framework(model_class) + if framework == "tf": + signature = inspect.signature(model_class.call) # TensorFlow models + elif framework == "pt": + signature = inspect.signature(model_class.forward) # PyTorch models + else: + signature = inspect.signature(model_class.__call__) # Flax models + + if "QuestionAnswering" in model_name: + return [p for p in signature.parameters if "label" in p or p in ("start_positions", "end_positions")] + else: + return [p for p in signature.parameters if "label" in p] + + +def flatten_dict(d: MutableMapping, parent_key: str = "", delimiter: str = "."): + """Flatten a nested dict into a single level dict.""" + + def _flatten_dict(d, parent_key="", delimiter="."): + for k, v in d.items(): + key = str(parent_key) + delimiter + str(k) if parent_key else k + if v and isinstance(v, MutableMapping): + yield from flatten_dict(v, key, delimiter=delimiter).items() + else: + yield key, v + + return dict(_flatten_dict(d, parent_key, delimiter)) + + +@contextmanager +def working_or_temp_dir(working_dir, use_temp_dir: bool = False): + if use_temp_dir: + with tempfile.TemporaryDirectory() as tmp_dir: + yield tmp_dir + else: + yield working_dir + + +def transpose(array, axes=None): + """ + Framework-agnostic version of `numpy.transpose` that will work on torch/TensorFlow/Jax tensors as well as NumPy + arrays. + """ + if is_numpy_array(array): + return np.transpose(array, axes=axes) + elif is_torch_tensor(array): + return array.T if axes is None else array.permute(*axes) + elif is_tf_tensor(array): + import tensorflow as tf + + return tf.transpose(array, perm=axes) + elif is_jax_tensor(array): + import jax.numpy as jnp + + return jnp.transpose(array, axes=axes) + else: + raise ValueError(f"Type not supported for transpose: {type(array)}.") + + +def reshape(array, newshape): + """ + Framework-agnostic version of `numpy.reshape` that will work on torch/TensorFlow/Jax tensors as well as NumPy + arrays. + """ + if is_numpy_array(array): + return np.reshape(array, newshape) + elif is_torch_tensor(array): + return array.reshape(*newshape) + elif is_tf_tensor(array): + import tensorflow as tf + + return tf.reshape(array, newshape) + elif is_jax_tensor(array): + import jax.numpy as jnp + + return jnp.reshape(array, newshape) + else: + raise ValueError(f"Type not supported for reshape: {type(array)}.") + + +def squeeze(array, axis=None): + """ + Framework-agnostic version of `numpy.squeeze` that will work on torch/TensorFlow/Jax tensors as well as NumPy + arrays. + """ + if is_numpy_array(array): + return np.squeeze(array, axis=axis) + elif is_torch_tensor(array): + return array.squeeze() if axis is None else array.squeeze(dim=axis) + elif is_tf_tensor(array): + import tensorflow as tf + + return tf.squeeze(array, axis=axis) + elif is_jax_tensor(array): + import jax.numpy as jnp + + return jnp.squeeze(array, axis=axis) + else: + raise ValueError(f"Type not supported for squeeze: {type(array)}.") + + +def expand_dims(array, axis): + """ + Framework-agnostic version of `numpy.expand_dims` that will work on torch/TensorFlow/Jax tensors as well as NumPy + arrays. + """ + if is_numpy_array(array): + return np.expand_dims(array, axis) + elif is_torch_tensor(array): + return array.unsqueeze(dim=axis) + elif is_tf_tensor(array): + import tensorflow as tf + + return tf.expand_dims(array, axis=axis) + elif is_jax_tensor(array): + import jax.numpy as jnp + + return jnp.expand_dims(array, axis=axis) + else: + raise ValueError(f"Type not supported for expand_dims: {type(array)}.") + + +def tensor_size(array): + """ + Framework-agnostic version of `numpy.size` that will work on torch/TensorFlow/Jax tensors as well as NumPy arrays. + """ + if is_numpy_array(array): + return np.size(array) + elif is_torch_tensor(array): + return array.numel() + elif is_tf_tensor(array): + import tensorflow as tf + + return tf.size(array) + elif is_jax_tensor(array): + return array.size + else: + raise ValueError(f"Type not supported for tensor_size: {type(array)}.") + + +def add_model_info_to_auto_map(auto_map, repo_id): + """ + Adds the information of the repo_id to a given auto map. + """ + for key, value in auto_map.items(): + if isinstance(value, (tuple, list)): + auto_map[key] = [f"{repo_id}--{v}" if (v is not None and "--" not in v) else v for v in value] + elif value is not None and "--" not in value: + auto_map[key] = f"{repo_id}--{value}" + + return auto_map + + +def add_model_info_to_custom_pipelines(custom_pipeline, repo_id): + """ + Adds the information of the repo_id to a given custom pipeline. + """ + # {custom_pipelines : {task: {"impl": "path.to.task"},...} } + for task in custom_pipeline.keys(): + if "impl" in custom_pipeline[task]: + module = custom_pipeline[task]["impl"] + if "--" not in module: + custom_pipeline[task]["impl"] = f"{repo_id}--{module}" + return custom_pipeline + + +def infer_framework(model_class): + """ + Infers the framework of a given model without using isinstance(), because we cannot guarantee that the relevant + classes are imported or available. + """ + for base_class in inspect.getmro(model_class): + module = base_class.__module__ + name = base_class.__name__ + if module.startswith("tensorflow") or module.startswith("keras") or name == "TFPreTrainedModel": + return "tf" + elif module.startswith("torch") or name == "PreTrainedModel": + return "pt" + elif module.startswith("flax") or module.startswith("jax") or name == "FlaxPreTrainedModel": + return "flax" + else: + raise TypeError(f"Could not infer framework from class {model_class}.") + + +def torch_int(x): + """ + Casts an input to a torch int64 tensor if we are in a tracing context, otherwise to a Python int. + """ + if not is_torch_available(): + return int(x) + + import torch + + return x.to(torch.int64) if torch.jit.is_tracing() and isinstance(x, torch.Tensor) else int(x) + + +def torch_float(x): + """ + Casts an input to a torch float32 tensor if we are in a tracing context, otherwise to a Python float. + """ + if not is_torch_available(): + return int(x) + + import torch + + return x.to(torch.float32) if torch.jit.is_tracing() and isinstance(x, torch.Tensor) else int(x) + + +def filter_out_non_signature_kwargs(extra: Optional[list] = None): + """ + Decorator to filter out named arguments that are not in the function signature. + + This decorator ensures that only the keyword arguments that match the function's signature, or are specified in the + `extra` list, are passed to the function. Any additional keyword arguments are filtered out and a warning is issued. + + Parameters: + extra (`Optional[list]`, *optional*): + A list of extra keyword argument names that are allowed even if they are not in the function's signature. + + Returns: + Callable: + A decorator that wraps the function and filters out invalid keyword arguments. + + Example usage: + + ```python + @filter_out_non_signature_kwargs(extra=["allowed_extra_arg"]) + def my_function(arg1, arg2, **kwargs): + print(arg1, arg2, kwargs) + + my_function(arg1=1, arg2=2, allowed_extra_arg=3, invalid_arg=4) + # This will print: 1 2 {"allowed_extra_arg": 3} + # And issue a warning: "The following named arguments are not valid for `my_function` and were ignored: 'invalid_arg'" + ``` + """ + extra = extra or [] + extra_params_to_pass = set(extra) + + def decorator(func): + sig = inspect.signature(func) + function_named_args = set(sig.parameters.keys()) + valid_kwargs_to_pass = function_named_args.union(extra_params_to_pass) + + # Required for better warning message + is_instance_method = "self" in function_named_args + is_class_method = "cls" in function_named_args + + # Mark function as decorated + func._filter_out_non_signature_kwargs = True + + @wraps(func) + def wrapper(*args, **kwargs): + valid_kwargs = {} + invalid_kwargs = {} + + for k, v in kwargs.items(): + if k in valid_kwargs_to_pass: + valid_kwargs[k] = v + else: + invalid_kwargs[k] = v + + if invalid_kwargs: + invalid_kwargs_names = [f"'{k}'" for k in invalid_kwargs.keys()] + invalid_kwargs_names = ", ".join(invalid_kwargs_names) + + # Get the class name for better warning message + if is_instance_method: + cls_prefix = args[0].__class__.__name__ + "." + elif is_class_method: + cls_prefix = args[0].__name__ + "." + else: + cls_prefix = "" + + warnings.warn( + f"The following named arguments are not valid for `{cls_prefix}{func.__name__}`" + f" and were ignored: {invalid_kwargs_names}", + UserWarning, + stacklevel=2, + ) + + return func(*args, **valid_kwargs) + + return wrapper + + return decorator + + +class LossKwargs(TypedDict, total=False): + """ + Keyword arguments to be passed to the loss function + + Attributes: + num_items_in_batch (`int`, *optional*): + Number of items in the batch. It is recommended to pass it when + you are doing gradient accumulation. + """ + + num_items_in_batch: Optional[int] + + +def is_timm_config_dict(config_dict: dict[str, Any]) -> bool: + """Checks whether a config dict is a timm config dict.""" + return "pretrained_cfg" in config_dict + + +def is_timm_local_checkpoint(pretrained_model_path: str) -> bool: + """ + Checks whether a checkpoint is a timm model checkpoint. + """ + if pretrained_model_path is None: + return False + + # in case it's Path, not str + pretrained_model_path = str(pretrained_model_path) + + is_file = os.path.isfile(pretrained_model_path) + is_dir = os.path.isdir(pretrained_model_path) + + # pretrained_model_path is a file + if is_file and pretrained_model_path.endswith(".json"): + with open(pretrained_model_path) as f: + config_dict = json.load(f) + return is_timm_config_dict(config_dict) + + # pretrained_model_path is a directory with a config.json + if is_dir and os.path.exists(os.path.join(pretrained_model_path, "config.json")): + with open(os.path.join(pretrained_model_path, "config.json")) as f: + config_dict = json.load(f) + return is_timm_config_dict(config_dict) + + return False + + +def set_attribute_for_modules(module: "torch.nn.Module", key: str, value: Any): + """ + Set a value to a module and all submodules. + """ + setattr(module, key, value) + for submodule in module.children(): + set_attribute_for_modules(submodule, key, value) + + +def del_attribute_from_modules(module: "torch.nn.Module", key: str): + """ + Delete a value from a module and all submodules. + """ + # because we might remove it previously in case it's a shared module, e.g. activation function + if hasattr(module, key): + delattr(module, key) + + for submodule in module.children(): + del_attribute_from_modules(submodule, key) + + +def can_return_tuple(func): + """ + Decorator to wrap model method, to call output.to_tuple() if return_dict=False passed as a kwarg or + use_return_dict=False is set in the config. + + Note: + output.to_tuple() convert output to tuple skipping all `None` values. + """ + + @wraps(func) + def wrapper(self, *args, **kwargs): + is_requested_to_return_tuple = kwargs.pop("return_dict", True) is False + is_configured_to_return_tuple = self.config.use_return_dict is False if hasattr(self, "config") else False + + # The following allows to convert output to tuple ONLY on top level forward call, + # while internal modules of the model will return Output objects + # to be able to use name-based attribute access in modeling code. + + # We will check if we are on top level module, if so, turn off to tuple conversion for all + # underling calls. + is_top_level_module = getattr(self, "_is_top_level_module", True) + if is_configured_to_return_tuple and is_top_level_module: + set_attribute_for_modules(self, "_is_top_level_module", False) + + try: + output = func(self, *args, **kwargs) + if is_requested_to_return_tuple or (is_configured_to_return_tuple and is_top_level_module): + output = output.to_tuple() + finally: + # Remove the flag after the model forward call is finished. + if is_configured_to_return_tuple and is_top_level_module: + del_attribute_from_modules(self, "_is_top_level_module") + + return output + + return wrapper diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/ggml.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/ggml.py new file mode 100644 index 0000000000000000000000000000000000000000..db7d93cad0427acd2c12683edd177bf28c1ceb07 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/ggml.py @@ -0,0 +1,692 @@ +# coding=utf-8 +# Copyright 2024 The ggml.ai team and The HuggingFace Inc. team. and pygguf author (github.com/99991) +# https://github.com/99991/pygguf +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf +with extra methods beings exposed +""" + +from array import array + +import numpy as np +from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors +from tokenizers.models import BPE, Unigram + +from .. import AddedToken +from ..convert_slow_tokenizer import GemmaConverter, GPT2Converter, LlamaConverter, Qwen2Converter, T5Converter +from ..utils import logging +from ..utils.logging import tqdm + + +logger = logging.get_logger(__name__) + + +GGUF_CONFIG_MAPPING = { + "general": { + "architecture": "model_type", + "name": "_model_name_or_path", + }, + "llama": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + # NOTE: rope.dimension_count==head_dim only suitable for llama/mistral + "rope.dimension_count": "head_dim", + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + }, + "mistral": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + # NOTE: rope.dimension_count==head_dim only suitable for llama/mistral + "rope.dimension_count": "head_dim", + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + }, + "qwen2": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + }, + "qwen2moe": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + "expert_count": "num_experts", + "expert_used_count": "num_experts_per_tok", + }, + "falcon": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + }, + "tokenizer": { + "ggml.bos_token_id": "bos_token_id", + "ggml.eos_token_id": "eos_token_id", + "ggml.unknown_token_id": "unk_token_id", + "ggml.padding_token_id": "pad_token_id", + }, + "phi3": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + }, + "bloom": { + "block_count": "n_layer", + "embedding_length": "hidden_size", + "attention.head_count": "n_head", + "vocab_size": "vocab_size", + "attention.layer_norm_epsilon": "layer_norm_epsilon", + }, + "t5": { + "context_length": "n_positions", + "block_count": "num_layers", + "feed_forward_length": "d_ff", + "embedding_length": "d_model", + "attention.key_length": "d_kv", + "attention.head_count": "num_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_epsilon": "layer_norm_epsilon", + "attention.relative_buckets_count": "relative_attention_num_buckets", + "decoder_start_token_id": "decoder_start_token_id", + "vocab_size": "vocab_size", + }, + "stablelm": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_epsilon": "layer_norm_eps", + "vocab_size": "vocab_size", + }, + "gpt2": { + "block_count": "n_layer", + "context_length": "n_ctx", + "embedding_length": "n_embd", + "feed_forward_length": "feed_forward_length", + "attention.head_count": "n_head", + "attention.layer_norm_epsilon": "layer_norm_epsilon", + }, + "starcoder2": { + "block_count": "num_hidden_layers", + "context_length": "max_position_embeddings", + "embedding_length": "hidden_size", + "feed_forward_length": "intermediate_size", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_epsilon": "norm_epsilon", + }, + "mamba": { + "vocab_size": "vocab_size", + "context_length": "max_position_embeddings", + "embedding_length": "hidden_size", + "attention.layer_norm_rms_epsilon": "layer_norm_epsilon", + "block_count": "num_hidden_layers", + "ssm.conv_kernel": "conv_kernel", + "ssm.state_size": "state_size", + "ssm.time_step_rank": "time_step_rank", + "ssm.inner_size": "intermediate_size", + }, + "nemotron": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "norm_eps", + "vocab_size": "vocab_size", + }, + "gemma2": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + # NOTE: Gemma2 has key_length==value_length==head_dim + # See: https://github.com/ggerganov/llama.cpp/blob/2e2f8f093cd4fb6bbb87ba84f6b9684fa082f3fa/convert_hf_to_gguf.py#L3293-L3294 + "attention.key_length": "head_dim", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + }, +} + +GGUF_TOKENIZER_MAPPING = { + "tokenizer": { + "ggml.model": "tokenizer_type", + "ggml.tokens": "tokens", + "ggml.scores": "scores", + "ggml.token_type": "token_type", + "ggml.merges": "merges", + "ggml.bos_token_id": "bos_token_id", + "ggml.eos_token_id": "eos_token_id", + "ggml.unknown_token_id": "unk_token_id", + "ggml.padding_token_id": "pad_token_id", + "ggml.add_space_prefix": "add_prefix_space", + }, + "tokenizer_config": { + "chat_template": "chat_template", + "ggml.model": "model_type", + "ggml.bos_token_id": "bos_token_id", + "ggml.eos_token_id": "eos_token_id", + "ggml.unknown_token_id": "unk_token_id", + "ggml.padding_token_id": "pad_token_id", + }, +} + + +def _gguf_parse_value(_value, data_type): + if not isinstance(data_type, list): + data_type = [data_type] + if len(data_type) == 1: + data_type = data_type[0] + array_data_type = None + else: + if data_type[0] != 9: + raise ValueError("Received multiple types, therefore expected the first type to indicate an array.") + data_type, array_data_type = data_type + + if data_type in [0, 1, 2, 3, 4, 5, 10, 11]: + _value = int(_value[0]) + elif data_type in [6, 12]: + _value = float(_value[0]) + elif data_type in [7]: + _value = bool(_value[0]) + elif data_type in [8]: + _value = array("B", list(_value)).tobytes().decode() + elif data_type in [9]: + _value = _gguf_parse_value(_value, array_data_type) + return _value + + +class GGUFTokenizerSkeleton: + def __init__(self, dict_): + for k, v in dict_.items(): + setattr(self, k, v) + + if not hasattr(self, "merges"): + if not hasattr(self, "tokens") or not hasattr(self, "scores"): + raise ValueError( + "tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated." + ) + tokens = self.tokens + scores = self.scores + vocab = {t: scores[i] for i, t in enumerate(tokens)} + + logger.warning("Merges were not in checkpoint, building merges on the fly.") + merges = [] + for merge, piece_score in tqdm(vocab.items()): + local = [] + for index in range(1, len(merge)): + piece_l, piece_r = merge[:index], merge[index:] + if piece_l in tokens and piece_r in tokens: + local.append((piece_l, piece_r, piece_score)) + local = sorted(local, key=lambda x: (vocab[x[0]], vocab[x[1]]), reverse=True) + merges.extend(local) + merges = sorted(merges, key=lambda val: val[2], reverse=True) + merges = [(val[0], val[1]) for val in merges] + self.merges = merges + else: + self.merges = [tuple(merge.split(" ")) for merge in self.merges] + if not hasattr(self, "scores"): + self.scores = [None for _ in range(len(self.tokens))] + + if not hasattr(self, "added_tokens"): + self.added_tokens = [] + + if not hasattr(self, "unk_token_id"): + self.unk_token_id = None + + # Llama2 uses the field `unknown_token_id` + if hasattr(self, "unknown_token_id") and self.unk_token_id is None: + self.unk_token_id = self.unknown_token_id + + +class GGUFLlamaConverter(LlamaConverter): + def __init__(self, tokenizer_dict): + self.proto = GGUFTokenizerSkeleton(tokenizer_dict) + self.original_tokenizer = self.proto + self.additional_kwargs = {} + self.is_llama_3_tokenizer = getattr(self.proto, "tokenizer_type", "llama") != "llama" + + def vocab(self, proto): + return list(zip(proto.tokens, proto.scores)) + + def merges(self, proto): + return proto.merges + + def tokenizer(self, proto): + vocab_scores = self.vocab(self.proto) + merges = self.merges(self.proto) + bpe_vocab = {word: i for i, (word, _score) in enumerate(vocab_scores)} + + unk_token = proto.tokens[proto.unk_token_id] if proto.unk_token_id is not None else None + bos_token = proto.tokens[proto.bos_token_id] if getattr(proto, "bos_token_id", None) is not None else None + eos_token = proto.tokens[proto.bos_token_id] if getattr(proto, "eos_token_id", None) is not None else None + + tokenizer = Tokenizer( + BPE( + bpe_vocab, + merges, + unk_token=unk_token, + fuse_unk=True, + byte_fallback=True, + ) + ) + + special_tokens = [] + + if not hasattr(self.proto, "token_type"): + if unk_token is not None: + special_tokens.append(AddedToken(unk_token, normalized=False, special=True)) + + if bos_token is not None: + special_tokens.append(AddedToken(bos_token, normalized=False, special=True)) + + if eos_token is not None: + special_tokens.append(AddedToken(eos_token, normalized=False, special=True)) + else: + # 3 stands for special tokens + special_tokens_idx = np.where(np.array(self.proto.token_type) == 3)[0] + + for idx in special_tokens_idx: + special_tokens.append(AddedToken(self.proto.tokens[idx], normalized=False, special=True)) + + if len(special_tokens) != 0: + tokenizer.add_special_tokens(special_tokens) + + if len(self.proto.added_tokens) != 0: + tokenizer.add_tokens( + [AddedToken(added_token, normalized=False, special=False) for added_token in self.proto.added_tokens] + ) + + self.additional_kwargs["unk_token"] = unk_token + self.additional_kwargs["eos_token"] = bos_token + self.additional_kwargs["bos_token"] = eos_token + + if self.is_llama_3_tokenizer: + self.additional_kwargs["add_prefix_space"] = None + self.additional_kwargs["clean_up_tokenization_spaces"] = True + + self.additional_kwargs["legacy"] = False + self.original_tokenizer.legacy = False + + return tokenizer + + def decoder(self, replacement, add_prefix_space): + sequence = [ + decoders.ByteFallback(), + decoders.Fuse(), + decoders.Replace("▁", " "), + ] + + if self.is_llama_3_tokenizer: + sequence += [decoders.ByteLevel(add_prefix_space=False, trim_offsets=False, use_regex=True)] + + if add_prefix_space: + sequence += [decoders.Strip(content=" ", left=1)] + return decoders.Sequence(sequence) + + def converted(self): + # Copied partly from converted method in SpmConverter class + tokenizer = self.tokenizer(self.proto) + + # Tokenizer assemble + normalizer = self.normalizer(self.proto) + if normalizer is not None: + tokenizer.normalizer = normalizer + + replacement = "▁" + add_prefix_space = True + if hasattr(self.original_tokenizer, "add_prefix_space"): + add_prefix_space = self.original_tokenizer.add_prefix_space + + pre_tokenizer = self.pre_tokenizer(replacement, add_prefix_space) + if pre_tokenizer is not None: + tokenizer.pre_tokenizer = pre_tokenizer + + tokenizer.decoder = self.decoder(replacement, add_prefix_space) + post_processor = self.post_processor() + if post_processor: + tokenizer.post_processor = post_processor + + # HACK: patch the llama-3 tokenizer to use the correspinding pre-tokenizer + # and normalizer + if self.is_llama_3_tokenizer: + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel( + add_prefix_space=False, trim_offsets=False, use_regex=True + ) + # This is tricky as the additional kwargs are passed after legacy is force-set in LlamaTokenizer's + # init. + tokenizer.normalizer = normalizers.Sequence([]) + + return tokenizer + + +class GGUFQwen2Converter(Qwen2Converter): + def __init__(self, tokenizer_dict): + self.original_tokenizer = GGUFTokenizerSkeleton(tokenizer_dict) + self.additional_kwargs = {} + + def converted(self) -> Tokenizer: + vocab = {word: i for i, word in enumerate(self.original_tokenizer.tokens)} + merges = self.original_tokenizer.merges + tokenizer = super().converted(vocab, merges) + + tokenizer.add_special_tokens( + [ + AddedToken("<|endoftext|>", normalized=False, special=True), + AddedToken("<|im_start|>", normalized=False, special=True), + AddedToken("<|im_end|>", normalized=False, special=True), + ] + ) + return tokenizer + + +class GGUFPhi3Converter(LlamaConverter): + def __init__(self, tokenizer_dict): + self.proto = GGUFTokenizerSkeleton(tokenizer_dict) + self.original_tokenizer = self.proto + self.additional_kwargs = {} + + def vocab(self, proto): + return list(zip(proto.tokens, proto.scores)) + + def merges(self, proto): + return proto.merges + + def tokenizer(self, proto): + vocab_scores = self.vocab(self.proto) + merges = self.merges(self.proto) + bpe_vocab = {word: i for i, (word, _score) in enumerate(vocab_scores)} + + tokenizer = Tokenizer(BPE(bpe_vocab, merges)) + # add the special tokens from phi3 tokenizer config + tokenizer.add_special_tokens( + [ + AddedToken("", rstrip=True, lstrip=False, normalized=False, special=True), + AddedToken("<|endoftext|>", normalized=False, special=True), + AddedToken("<|assistant|>", rstrip=True, normalized=False, special=True), + AddedToken("<|placeholder1|>", rstrip=True, normalized=False, special=True), + AddedToken("<|placeholder2|>", rstrip=True, normalized=False, special=True), + AddedToken("<|placeholder3|>", rstrip=True, normalized=False, special=True), + AddedToken("<|placeholder4|>", rstrip=True, normalized=False, special=True), + AddedToken("<|system|>", rstrip=True, normalized=False, special=True), + AddedToken("<|end|>", rstrip=True, normalized=False, special=True), + AddedToken("<|placeholder5|>", rstrip=True, normalized=False, special=True), + AddedToken("<|placeholder6|>", rstrip=True, normalized=False, special=True), + AddedToken("<|user|>", rstrip=True, normalized=False, special=True), + ] + ) + + self.additional_kwargs["unk_token"] = ( + proto.tokens[proto.unk_token_id] if proto.unk_token_id is not None else None + ) + self.additional_kwargs["eos_token"] = ( + proto.tokens[proto.eos_token_id] if proto.eos_token_id is not None else None + ) + self.additional_kwargs["bos_token"] = ( + proto.tokens[proto.bos_token_id] if proto.bos_token_id is not None else None + ) + self.additional_kwargs["pad_token"] = ( + proto.tokens[proto.pad_token_id] if proto.pad_token_id is not None else None + ) + + return tokenizer + + def decoder(self, replacement, add_prefix_space): + sequence = [ + decoders.ByteFallback(), + decoders.Fuse(), + decoders.Replace(replacement, " "), + ] + + if add_prefix_space: + sequence += [decoders.Strip(content=" ", left=1)] + return decoders.Sequence(sequence) + + def converted(self) -> Tokenizer: + tokenizer = self.tokenizer(self.proto) + + replacement = "▁" + add_prefix_space = True + if hasattr(self.original_tokenizer, "add_prefix_space"): + add_prefix_space = self.original_tokenizer.add_prefix_space + + tokenizer.decoder = self.decoder(replacement, add_prefix_space) + + return tokenizer + + +class GGUFGPTConverter(GPT2Converter): + def __init__(self, tokenizer_dict): + self.original_tokenizer = GGUFTokenizerSkeleton(tokenizer_dict) + self.additional_kwargs = {} + + def converted(self) -> Tokenizer: + vocab = {word: i for i, word in enumerate(self.original_tokenizer.tokens)} + merges = self.original_tokenizer.merges + tokenizer = super().converted(vocab, merges) + return tokenizer + + +class GGUFT5Converter(T5Converter): + def __init__(self, tokenizer_dict): + # set dummy data to avoid unnecessary merges calculation + tokenizer_dict["merges"] = ["dummy text"] + + self.proto = GGUFTokenizerSkeleton(tokenizer_dict) + self.token2id = {k: v for v, k in enumerate(self.proto.tokens)} + self.original_tokenizer = self.proto + self.additional_kwargs = {} + + def vocab(self, proto): + return list(zip(proto.tokens, proto.scores)) + + def normalizer(self, proto): + if getattr(self.original_tokenizer, "legacy", True): + sequence = [] + if getattr(self.original_tokenizer, "add_prefix_space", True): + sequence += [normalizers.Prepend(prepend="▁")] + sequence += [normalizers.Replace(pattern=" ", content="▁")] + return normalizers.Sequence(sequence) + return None # non-legacy, no normalizer + + def post_processor(self): + return processors.TemplateProcessing( + single=["$A", ""], + pair=["$A", "", "$B", ""], + special_tokens=[ + ("", self.token2id[""]), + ], + ) + + def converted(self) -> Tokenizer: + vocab_scores = self.vocab(self.proto) + tokenizer = Tokenizer( + Unigram( + vocab_scores, + unk_id=self.proto.unk_token_id, + byte_fallback=False, + ) + ) + + # Tokenizer assemble + normalizer = self.normalizer(self.proto) + if normalizer is not None: + tokenizer.normalizer = normalizer + + replacement = "▁" + add_prefix_space = True + if hasattr(self.original_tokenizer, "add_prefix_space"): + add_prefix_space = self.original_tokenizer.add_prefix_space + + pre_tokenizer = self.pre_tokenizer(replacement, add_prefix_space) + if pre_tokenizer is not None: + tokenizer.pre_tokenizer = pre_tokenizer + + tokenizer.decoder = self.decoder(replacement, add_prefix_space) + post_processor = self.post_processor() + if post_processor: + tokenizer.post_processor = post_processor + + return tokenizer + + +class GGUFGemmaConverter(GemmaConverter): + def __init__(self, tokenizer_dict): + # set dummy data to avoid unnecessary merges calculation + tokenizer_dict["merges"] = ["dummy text"] + + self.proto = GGUFTokenizerSkeleton(tokenizer_dict) + self.original_tokenizer = self.proto + self.additional_kwargs = {} + + def vocab(self, proto): + original_vocab = list(zip(proto.tokens, proto.scores)) + updated_vocab = [] + + for token, score in original_vocab: + if token == "<0x09>": + updated_vocab.append(("\t", score)) + elif " " in token and len(token.strip()) == 0: + underscores = "▁" * len(token) + updated_vocab.append((underscores, score)) + else: + updated_vocab.append((token, score)) + + return updated_vocab + + def normalizer(self, proto): + return normalizers.Replace(" ", "▁") + + def decoder(self, replacement, add_prefix_space): + sequence = [ + decoders.Replace("▁", " "), + decoders.ByteFallback(), + decoders.Fuse(), + ] + + if add_prefix_space: + sequence += [decoders.Strip(content=" ", left=1)] + return decoders.Sequence(sequence) + + def converted(self) -> Tokenizer: + vocab_scores = self.vocab(self.proto) + tokenizer = Tokenizer( + Unigram( + vocab_scores, + unk_id=self.proto.unk_token_id, + byte_fallback=self.handle_byte_fallback, + ) + ) + + normalizer = self.normalizer(self.proto) + if normalizer is not None: + tokenizer.normalizer = normalizer + + replacement = "▁" + add_prefix_space = True + if hasattr(self.original_tokenizer, "add_prefix_space"): + add_prefix_space = self.original_tokenizer.add_prefix_space + + tokenizer.decoder = self.decoder(replacement, add_prefix_space) + pre_tokenizer = self.pre_tokenizer(replacement, add_prefix_space) + if pre_tokenizer is not None: + tokenizer.pre_tokenizer = pre_tokenizer + + return tokenizer + + +GGUF_TO_FAST_CONVERTERS = { + "llama": GGUFLlamaConverter, + "qwen2": GGUFQwen2Converter, + "qwen2_moe": GGUFQwen2Converter, + "phi3": GGUFPhi3Converter, + "bloom": GGUFGPTConverter, + "falcon": GGUFGPTConverter, + "stablelm": GGUFGPTConverter, + "gpt2": GGUFGPTConverter, + "starcoder2": GGUFGPTConverter, + "t5": GGUFT5Converter, + "mamba": GGUFGPTConverter, + "nemotron": GGUFGPTConverter, + "gemma2": GGUFGemmaConverter, +} + + +def convert_gguf_tokenizer(architecture, tokenizer_dict) -> Tokenizer: + """ + Utilities to convert a slow tokenizer instance in a fast tokenizer instance. + + Args: + architecture (`str`): The model architecture derived from gguf file. + transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]): + Instance of a slow tokenizer to convert in the backend tokenizer for + [`~tokenization_utils_base.PreTrainedTokenizerFast`]. + + Return: + A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a + [`~tokenization_utils_base.PreTrainedTokenizerFast`] + """ + tokenizer_class_name = architecture + converter = GGUF_TO_FAST_CONVERTERS[tokenizer_class_name](tokenizer_dict) + fast_tokenizer = converter.converted() + return fast_tokenizer, converter.additional_kwargs diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/higgs.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/higgs.py new file mode 100644 index 0000000000000000000000000000000000000000..dd31764dfe0ce731077a3fca9c6e35579c65b50b --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/higgs.py @@ -0,0 +1,655 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"HIGGS through FLUTE (Flexible Lookup Table Engine for LUT-quantized LLMs) integration file" + +from math import sqrt + +from ..utils import ( + is_flute_available, + is_hadamard_available, + is_torch_available, +) + + +if is_torch_available(): + import torch + from torch import nn + + +if is_flute_available(): + from flute.integrations.higgs import prepare_data_transposed + from flute.tune import TuneMetaData, qgemm_v2 + +if is_hadamard_available(): + from fast_hadamard_transform import hadamard_transform + + +def pad_to_block(tensor, dims, had_block_size, value=0): + pad_dims = [0 for _ in range(2 * len(tensor.shape))] + for dim in dims: + size = tensor.shape[dim] + next_multiple_of_1024 = ((size - 1) // had_block_size + 1) * had_block_size + delta = next_multiple_of_1024 - size + pad_dims[-2 * dim - 1] = delta + + return nn.functional.pad(tensor, pad_dims, "constant", value) + + +def get_higgs_grid(p: int, n: int): + if (p, n) == (2, 256): + return torch.tensor( + [ + [-2.501467704772949, 0.17954708635807037], + [-0.6761789321899414, 1.2728623151779175], + [-1.8025816679000854, 0.7613157629966736], + [-0.538287878036499, -2.6028504371643066], + [0.8415029644966125, -0.8600977659225464], + [0.7023013234138489, 3.3138747215270996], + [0.5699077844619751, 2.5782253742218018], + [3.292393207550049, -0.6016128063201904], + [0.5561617016792297, -1.7723814249038696], + [-2.1012380123138428, 0.020958125591278076], + [0.46085724234580994, 0.8428705334663391], + [1.4548040628433228, -0.6156039237976074], + [3.210029363632202, 0.3546904921531677], + [0.8893890976905823, -0.5967988967895508], + [0.8618854284286499, -3.2061192989349365], + [1.1360996961593628, -0.23852407932281494], + [1.6646337509155273, -0.9265465140342712], + [1.4767773151397705, 1.2476022243499756], + [-1.0511897802352905, 1.94503915309906], + [-1.56318998336792, -0.3264186680316925], + [-0.1829211413860321, 0.2922491431236267], + [-0.8950616717338562, -1.3887052536010742], + [-0.08206957578659058, -1.329533576965332], + [-0.487422913312912, 1.4817842245101929], + [-1.6769757270812988, -2.8269758224487305], + [-1.5057679414749146, 1.8905963897705078], + [1.8335362672805786, 1.0515104532241821], + [0.3273945450782776, 1.0491033792495728], + [-3.295924186706543, -0.7021600008010864], + [-1.8428784608840942, -1.2315762042999268], + [-0.8575026392936707, -1.7005949020385742], + [-1.120667815208435, 0.6467998027801514], + [-0.1588846743106842, -1.804071068763733], + [-0.8539647459983826, 0.5645008683204651], + [-1.4192019701004028, -0.6175029873847961], + [1.0799058675765991, 1.7871345281600952], + [1.171311855316162, 0.7511613965034485], + [2.162078380584717, 0.8044339418411255], + [1.3969420194625854, -1.243762493133545], + [-0.23818807303905487, 0.053944624960422516], + [2.304199457168579, -1.2667627334594727], + [1.4225027561187744, 0.568610668182373], + [0.376836895942688, -0.7134661674499512], + [2.0404467582702637, 0.4087389409542084], + [0.7639489769935608, -1.1367933750152588], + [0.3622530400753021, -1.4827953577041626], + [0.4100743532180786, 0.36108437180519104], + [-1.5867475271224976, -1.618212342262268], + [-2.2769672870635986, -1.2132309675216675], + [0.9184022545814514, -0.34428009390830994], + [-0.3902314603328705, 0.21785245835781097], + [3.120687484741211, 1.3077973127365112], + [1.587440848350525, -1.6506884098052979], + [-1.718808889389038, -0.038405973464250565], + [-0.6888407468795776, -0.8402308821678162], + [-0.7981445789337158, -1.1117373704910278], + [-2.4124443531036377, 1.3419722318649292], + [-0.6611530184745789, 0.9939885139465332], + [-0.33103418350219727, -0.16702833771705627], + [-2.4091389179229736, -2.326857566833496], + [1.6610108613967896, -2.159703254699707], + [0.014884627424180508, 0.3887578248977661], + [0.029668325558304787, 1.8786455392837524], + [1.180362582206726, 2.699317216873169], + [1.821286678314209, -0.5960053205490112], + [-0.44835323095321655, 3.327436685562134], + [-0.3714401423931122, -2.1466753482818604], + [-1.1103475093841553, -2.4536871910095215], + [-0.39110705256462097, 0.6670510172843933], + [0.474752813577652, -1.1959707736968994], + [-0.013110585510730743, -2.52519154548645], + [-2.0836575031280518, -1.703289270401001], + [-1.1077687740325928, -0.1252644956111908], + [-0.4138077199459076, 1.1837692260742188], + [-1.977599024772644, 1.688241720199585], + [-1.659559965133667, -2.1387736797332764], + [0.03242531046271324, 0.6526556015014648], + [0.9127950072288513, 0.6099498867988586], + [-0.38478314876556396, 0.433487206697464], + [0.27454206347465515, -0.27719801664352417], + [0.10388526320457458, 2.2812814712524414], + [-0.014394169673323631, -3.177137613296509], + [-1.2871228456497192, -0.8961855173110962], + [0.5720916986465454, -0.921597957611084], + [1.1159656047821045, -0.7609877586364746], + [2.4383342266082764, -2.2983546257019043], + [-0.294057160615921, -0.9770799875259399], + [-0.9342701435089111, 1.107579231262207], + [-1.549338698387146, 3.090520143508911], + [2.6076579093933105, 2.051239013671875], + [-0.9259037375450134, 1.407211184501648], + [-0.1747353971004486, 0.540488600730896], + [-0.8963701725006104, 0.8271111249923706], + [0.6480194926261902, 1.0128909349441528], + [0.980783998966217, -0.06156221032142639], + [-0.16883476078510284, 1.0601658821105957], + [0.5839992761611938, 0.004697148688137531], + [-0.34228450059890747, -1.2423977851867676], + [2.500824451446533, 0.3665279746055603], + [-0.17641609907150269, 1.3529551029205322], + [0.05378641560673714, 2.817232847213745], + [-1.2391047477722168, 2.354328155517578], + [0.630434513092041, -0.668536365032196], + [1.7576488256454468, 0.6738647818565369], + [0.4435231387615204, 0.6000469326972961], + [-0.08794835954904556, -0.11511358618736267], + [1.6540337800979614, 0.33995017409324646], + [-0.04202975332736969, -0.5375117063522339], + [-0.4247745871543884, -0.7897617220878601], + [0.06695003807544708, 1.2000739574432373], + [-3.2508881092071533, 0.28734830021858215], + [-1.613816261291504, 0.4944162368774414], + [1.3598989248275757, 0.26117825508117676], + [2.308382511138916, 1.3462618589401245], + [-1.2137469053268433, -1.9254342317581177], + [-0.4889402985572815, 1.8136259317398071], + [-0.1870335340499878, -0.3480615019798279], + [1.0766386985778809, -1.0627082586288452], + [0.4651014506816864, 2.131748914718628], + [-0.1306295394897461, -0.7811847925186157], + [0.06433182954788208, -1.5397958755493164], + [-0.2894323468208313, -0.5789554715156555], + [-0.6081662178039551, 0.4845278263092041], + [2.697964668273926, -0.18515698611736298], + [0.1277363896369934, -0.7221432328224182], + [0.8700758218765259, 0.35042452812194824], + [0.22088994085788727, 0.495242178440094], + [-2.5843818187713623, -0.8000828623771667], + [0.6732649803161621, -1.4362232685089111], + [-1.5286413431167603, 1.0417330265045166], + [-1.1222513914108276, -0.6269875764846802], + [-0.9752035140991211, -0.8750635385513306], + [-2.6369473934173584, 0.6918523907661438], + [0.14478731155395508, -0.041986867785453796], + [-1.5629483461380005, 1.4369450807571411], + [0.38952457904815674, -2.16428804397583], + [-0.16885095834732056, 0.7976621985435486], + [-3.12416934967041, 1.256506085395813], + [0.6843105554580688, -0.4203019142150879], + [1.9345275163650513, 1.934950351715088], + [0.012184220366179943, -2.1080918312072754], + [-0.6350273489952087, 0.7358828186988831], + [-0.837304949760437, -0.6214472651481628], + [0.08211923390626907, -0.9472538232803345], + [2.9332995414733887, -1.4956780672073364], + [1.3806978464126587, -0.2916182279586792], + [0.06773144006729126, 0.9285762310028076], + [-1.1943119764328003, 1.5963770151138306], + [1.6395620107650757, -0.32285431027412415], + [-1.390851378440857, -0.08273141086101532], + [1.816330909729004, -1.2812227010726929], + [0.7921574711799622, -2.1135804653167725], + [0.5817914605140686, 1.2644577026367188], + [1.929347038269043, -0.2386285960674286], + [0.8877345323562622, 1.190008521080017], + [1.4732073545455933, 0.8935023546218872], + [-2.8518524169921875, -1.5478795766830444], + [0.2439267635345459, 0.7576767802238464], + [0.5246709585189819, -2.606659412384033], + [1.150876760482788, 1.4073830842971802], + [-0.2643202245235443, 2.0634236335754395], + [1.555483341217041, -0.0023102816194295883], + [2.0830578804016113, -1.7225427627563477], + [-0.5424830317497253, -1.070199728012085], + [0.9168899655342102, 0.8955540060997009], + [-0.8120972514152527, 2.696739912033081], + [-0.29908373951911926, -1.5310651063919067], + [1.2320337295532227, -1.556247353553772], + [1.8612544536590576, 0.08704725652933121], + [0.22133447229862213, -1.8091708421707153], + [-0.4403655230998993, -0.38571012020111084], + [-1.88539457321167, 1.192205786705017], + [2.239687919616699, 0.004709010478109121], + [1.139495611190796, 0.45733731985092163], + [-1.507995367050171, 0.19716016948223114], + [0.46986445784568787, 1.5422041416168213], + [-1.2573751211166382, -0.35984551906585693], + [-1.7415345907211304, -0.6020717024803162], + [1.0751984119415283, 0.19006384909152985], + [2.24186635017395, -0.46343153715133667], + [0.3610347509384155, -0.07658443599939346], + [-1.3111497163772583, 0.432013601064682], + [0.6164408326148987, 0.24538464844226837], + [-1.9266542196273804, -0.3256155550479889], + [-0.5870336890220642, -0.1879584938287735], + [-1.0476511716842651, 0.3677721917629242], + [-1.229940414428711, 1.2433830499649048], + [0.18550436198711395, 0.22753673791885376], + [-0.017921989783644676, 0.12625974416732788], + [1.1659504175186157, -0.5020995736122131], + [-0.5983408093452454, -1.40438973903656], + [0.7519024014472961, -0.16282692551612854], + [0.9920787811279297, -1.344896912574768], + [-0.8103678226470947, 0.3064485788345337], + [0.6956969499588013, 1.8208192586898804], + [-2.7830491065979004, -0.2299390584230423], + [-0.34681546688079834, 2.4890666007995605], + [-1.4452646970748901, -1.2216600179672241], + [-2.1872897148132324, 0.8926076292991638], + [1.706072211265564, -2.8440372943878174], + [1.1119003295898438, -2.4923460483551025], + [-2.582794666290283, 2.0973289012908936], + [0.04987720400094986, -0.2964983284473419], + [-2.063807487487793, -0.7847916483879089], + [-0.4068813621997833, 0.9135897755622864], + [-0.9814359545707703, -0.3874954879283905], + [-1.4227229356765747, 0.7337291240692139], + [0.3065044581890106, 1.3125417232513428], + [1.2160996198654175, -1.9643305540084839], + [-1.2163853645324707, 0.14608727395534515], + [-2.3030710220336914, -0.37558120489120483], + [0.9232977628707886, 2.1843791007995605], + [-0.1989777386188507, 1.651851773262024], + [-0.714374840259552, -0.39365994930267334], + [-0.7805715799331665, -2.099881887435913], + [0.9015759229660034, -1.7053706645965576], + [0.1033422127366066, 1.5256654024124146], + [-1.8773194551467896, 2.324174165725708], + [1.9227174520492554, 2.7441604137420654], + [-0.5994020104408264, 0.23984014987945557], + [1.3496100902557373, -0.9126054644584656], + [-0.8765304088592529, -3.1877026557922363], + [-1.2040035724639893, -1.5169521570205688], + [1.4261796474456787, 2.150200128555298], + [1.463774561882019, 1.6656692028045654], + [0.20364105701446533, -0.4988172650337219], + [0.5195154547691345, -0.24067887663841248], + [-1.1116786003112793, -1.1599653959274292], + [-0.8490808606147766, -0.1681060940027237], + [0.3189965784549713, -0.9641751646995544], + [-0.5664751529693604, -0.5951744318008423], + [-1.6347930431365967, -0.9137664437294006], + [0.44048091769218445, -0.47259435057640076], + [-2.147747039794922, 0.47442489862442017], + [1.834734320640564, 1.4462147951126099], + [1.1777573823928833, 1.0659226179122925], + [-0.9568989872932434, 0.09495053440332413], + [-1.838529348373413, 0.2950586676597595], + [-0.4800611734390259, 0.014894310384988785], + [-0.5235516428947449, -1.7687653303146362], + [2.0735011100769043, -0.8825281262397766], + [2.637502431869507, 0.8455678224563599], + [2.606602907180786, -0.7848446369171143], + [-1.1886937618255615, 0.9330510497093201], + [0.38082656264305115, 0.13328030705451965], + [0.6847941875457764, 0.7384101152420044], + [1.2638574838638306, -0.007309418171644211], + [0.18292222917079926, -1.22371244430542], + [0.8143821954727173, 1.4976691007614136], + [0.6571850776672363, 0.48368802666664124], + [-0.6991601586341858, 2.150190830230713], + [0.8101756572723389, 0.10206498205661774], + [-0.08768226951360703, -1.084917664527893], + [-0.7208092212677002, 0.03657956421375275], + [0.3211449086666107, 1.803687334060669], + [-0.7835946083068848, 1.6869111061096191], + ] + ) + if (p, n) == (2, 64): + return torch.tensor( + [ + [-2.7216711044311523, 0.14431366324424744], + [-0.766914427280426, 1.7193410396575928], + [-2.2575762271881104, 1.2476624250411987], + [1.233758807182312, -2.3560616970062256], + [0.8701965808868408, -0.2649352252483368], + [1.4506438970565796, 2.1776366233825684], + [-0.06305818259716034, 1.9049758911132812], + [2.536226511001587, 0.563927412033081], + [0.4599496126174927, -1.8745561838150024], + [-1.900517225265503, -0.30703988671302795], + [0.09386251866817474, 0.8755807280540466], + [1.946500539779663, -0.6743080615997314], + [2.1338934898376465, 1.4581491947174072], + [0.9429940581321716, -0.8038390278816223], + [2.0697755813598633, -1.614896535873413], + [0.772676408290863, 0.22017823159694672], + [1.0689979791641235, -1.525044322013855], + [0.6813604831695557, 1.1345642805099487], + [0.4706456661224365, 2.606626272201538], + [-1.294018030166626, -0.4372096061706543], + [-0.09134224057197571, 0.4610418677330017], + [-0.7907772064208984, -0.48412787914276123], + [0.060459110885858536, -0.9172890186309814], + [-0.5855047702789307, 2.56172513961792], + [0.11484206467866898, -2.659848213195801], + [-1.5893300771713257, 2.188580274581909], + [1.6750942468643188, 0.7089915871620178], + [-0.445697546005249, 0.7452405095100403], + [-1.8539940118789673, -1.8377939462661743], + [-1.5791912078857422, -1.017285943031311], + [-1.030419945716858, -1.5746369361877441], + [-1.9511750936508179, 0.43696075677871704], + [-0.3446580767631531, -1.8953213691711426], + [-1.4219647645950317, 0.7676230669021606], + [-0.9191089272499084, 0.5021472573280334], + [0.20464491844177246, 1.3684605360031128], + [0.5402919054031372, 0.6699410676956177], + [1.8903915882110596, 0.03638288006186485], + [0.4723062515258789, -0.6216739416122437], + [-0.41345009207725525, -0.22752176225185394], + [2.7119064331054688, -0.5111885070800781], + [1.065286636352539, 0.6950305700302124], + [0.40629103779792786, -0.14339995384216309], + [1.2815024852752686, 0.17108257114887238], + [0.01785222627222538, -0.43778058886528015], + [0.054590027779340744, -1.4225547313690186], + [0.3076786696910858, 0.30697619915008545], + [-0.9498570561408997, -0.9576997756958008], + [-2.4640724658966064, -0.9660449028015137], + [1.3714425563812256, -0.39760473370552063], + [-0.4857747256755829, 0.2386789172887802], + [1.2797833681106567, 1.3097363710403442], + [0.5508887767791748, -1.1777795553207397], + [-1.384316325187683, 0.1465839296579361], + [-0.46556955575942993, -1.2442727088928223], + [-0.3915477693080902, -0.7319604158401489], + [-1.4005504846572876, 1.3890998363494873], + [-0.8647305965423584, 1.0617644786834717], + [-0.8901953101158142, -0.01650036871433258], + [-0.9893633723258972, -2.4662880897521973], + [1.445534110069275, -1.049334168434143], + [-0.041650623083114624, 0.012734669260680676], + [-0.3302375078201294, 1.26217782497406], + [0.6934980154037476, 1.7714335918426514], + ] + ) + elif (p, n) == (2, 16): + return torch.tensor( + [ + [-0.8996632695198059, -1.6360418796539307], + [-0.961183488368988, 1.5999565124511719], + [-1.882026195526123, 0.678778350353241], + [0.36300793290138245, -1.9667866230010986], + [-0.6814072728157043, -0.576818585395813], + [0.7270012497901917, 0.6186859607696533], + [0.3359416127204895, 1.8371193408966064], + [1.859930396080017, 0.036668598651885986], + [0.17208248376846313, -0.9401724338531494], + [-1.7599700689315796, -0.6244229674339294], + [-0.8993809223175049, 0.32267823815345764], + [0.839488685131073, -0.3017036020755768], + [1.5314953327178955, 1.2942044734954834], + [-0.0011779458727687597, 0.00022069070837460458], + [1.4274526834487915, -1.207889199256897], + [-0.16123905777931213, 0.8787511587142944], + ] + ) + elif (p, n) == (1, 16): + return torch.tensor( + [ + [-2.7325894832611084], + [-2.069017171859741], + [-1.6180464029312134], + [-1.2562311887741089], + [-0.9423404335975647], + [-0.6567591428756714], + [-0.38804829120635986], + [-0.12839503586292267], + [0.12839503586292267], + [0.38804829120635986], + [0.6567591428756714], + [0.9423404335975647], + [1.2562311887741089], + [1.6180464029312134], + [2.069017171859741], + [2.7325894832611084], + ] + ) + elif (p, n) == (1, 8): + return torch.tensor( + [ + [-2.1519455909729004], + [-1.3439092636108398], + [-0.7560052871704102], + [-0.2450941801071167], + [0.2450941801071167], + [0.7560052871704102], + [1.3439092636108398], + [2.1519455909729004], + ] + ) + elif (p, n) == (1, 4): + return torch.tensor([[-1.5104175806045532], [-0.4527800381183624], [0.4527800381183624], [1.5104175806045532]]) + else: + raise NotImplementedError(f"Unsupported p={p}, n={n}") + + +def quantize_with_higgs(weight, bits: int = 4, p: int = 2, group_size: int = 256, hadamard_size: int = 1024): + assert len(weight.shape) == 2, "Only 2D weights are supported for now" + + grid = get_higgs_grid(p, 2 ** (p * bits)).to(weight.device) + grid_norm_2 = torch.linalg.norm(grid, axis=-1) ** 2 + + device = weight.device + dtype = weight.dtype + weight = weight.to(copy=True, dtype=torch.float32) + # Pad to Hadamard transform size + weight = pad_to_block(weight, [1], hadamard_size) + + # Scale and Hadamard transform + mult = weight.shape[1] // hadamard_size + weight = weight.reshape(-1, mult, hadamard_size) + scales = torch.linalg.norm(weight, axis=-1) + weight = hadamard_transform(weight, 1) / scales[:, :, None] + + # Pad to edenn_d and project + weight = pad_to_block(weight, [2], p).reshape(weight.shape[0], mult, -1, p) + + # Quantize + codes = torch.empty(weight.shape[:-1], device=device, dtype=torch.uint8) + for i in range(0, weight.shape[0], 16): + codes[i : i + 16] = torch.argmax(2 * weight[i : i + 16] @ grid.T - grid_norm_2, dim=-1).to(torch.uint8) + del weight + + codes = codes.reshape(codes.shape[0], -1) + scales = scales / sqrt(hadamard_size) + + weight, scales, tables, tables2, tune_metadata = prepare_data_transposed( + codes, + torch.repeat_interleave(scales.to(dtype), hadamard_size // group_size, dim=1), + grid.to(dtype), + num_bits=bits, + group_size=group_size, + vector_size=p, + dtype=dtype, + device=device, + check_correctness=False, + ) + + return { + "weight": weight, + "scales": scales, + "tables": tables, + "tables2": tables2.view(dtype=torch.float16), + "tune_metadata": tune_metadata, + } + + +class HiggsLinear(torch.nn.Module): + def __init__( + self, + in_features: int, + out_features: int, + num_bits: int, + bias=True, + dtype: torch.dtype = None, + device: torch.device = None, + group_size: int = 256, + hadamard_size: int = 1024, + ): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.num_bits = num_bits + self.group_size = group_size + self.hadamard_size = hadamard_size + + assert in_features % group_size == 0 + assert num_bits in [2, 3, 4] + + self.weight = nn.Parameter( + torch.empty((out_features * num_bits // 16, in_features), dtype=torch.int16, device=device), + requires_grad=False, + ) + self.scales = nn.Parameter( + torch.empty((out_features, in_features // group_size), dtype=dtype, device=device), requires_grad=False + ) + self.tables = nn.Parameter(torch.empty((2**num_bits,), dtype=dtype, device=device), requires_grad=False) + self.tables2 = nn.Parameter( + torch.empty((2**num_bits, 2**num_bits, 2), dtype=dtype, device=device), requires_grad=False + ) + + if bias: + self.bias = nn.Parameter(torch.empty(out_features, device=device, dtype=dtype), requires_grad=False) + else: + self.register_parameter("bias", None) + + self.workspace = None # must be set externally to be reused among layers + self.tune_metadata: TuneMetaData = None # must be set externally because architecture dependent + + def forward(self, x): + x = pad_to_block(x, [-1], self.hadamard_size) + + if self.workspace is None: + raise Exception("Workspace must be set before calling forward") + + return qgemm_v2( + x, + self.weight, + self.scales, + self.tables, + self.tables2.view(dtype=torch.float32), + self.workspace, + self.tune_metadata, + hadamard_size=self.hadamard_size, + ) + + +def replace_with_higgs_linear( + model, + quantization_config=None, + current_key_name=None, + has_been_replaced=False, +): + """ + Public method that recursively replaces the Linear layers of the given model with HIGGS quantized layers. + `accelerate` is needed to use this method. Returns the converted model and a boolean that indicates if the + conversion has been successfull or not. + + Args: + model (`torch.nn.Module`): + The model to convert, can be any `torch.nn.Module` instance. + quantization_config (`HiggsConfig`): + The quantization config object that contains the quantization parameters. + current_key_name (`list`, *optional*): + A list that contains the current key name. This is used for recursion and should not be passed by the user. + has_been_replaced (`bool`, *optional*): + A boolean that indicates if the conversion has been successful or not. This is used for recursion and + should not be passed by the user. + """ + + from accelerate import init_empty_weights + + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + if isinstance(module, nn.Linear): + # Check if the current key is not in the `quantization_config.modules_to_not_convert` + current_key_name_str = ".".join(current_key_name) + if not any(current_key_name_str.endswith(key) for key in quantization_config.modules_to_not_convert): + with init_empty_weights(): + in_features = module.in_features + out_features = module.out_features + + model._modules[name] = HiggsLinear( + in_features, + out_features, + bias=module.bias is not None, + num_bits=quantization_config.bits, + hadamard_size=quantization_config.hadamard_size, + group_size=quantization_config.group_size, + ) + has_been_replaced = True + + # Store the module class in case we need to transpose the weight later + model._modules[name].source_cls = type(module) + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) + if len(list(module.children())) > 0: + _, has_been_replaced = replace_with_higgs_linear( + module, + quantization_config=quantization_config, + current_key_name=current_key_name, + has_been_replaced=has_been_replaced, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def dequantize_higgs(model, current_key_name=None): + """ + Dequantizes the HiggsLinear layers in the given model by replacing them with standard torch.nn.Linear layers. + Args: + model (torch.nn.Module): The model containing HiggsLinear layers to be dequantized. + current_key_name (list, optional): A list to keep track of the current module names during recursion. Defaults to None. + Returns: + torch.nn.Module: The model with HiggsLinear layers replaced by torch.nn.Linear layers. + """ + + with torch.no_grad(): + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + if isinstance(module, HiggsLinear): + in_features = module.in_features + out_features = module.out_features + + model._modules[name] = torch.nn.Linear( + in_features, + out_features, + bias=module.bias is not None, + device=module.scales.device, + dtype=module.scales.dtype, + ) + + model._modules[name].weight.data = module( + torch.eye(in_features, device=module.scales.device, dtype=module.scales.dtype) + ).T.contiguous() + + if len(list(module.children())) > 0: + _ = dequantize_higgs( + module, + current_key_name=current_key_name, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + return model diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hqq.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hqq.py new file mode 100644 index 0000000000000000000000000000000000000000..4ff154ee2056c26d5e9005c7fa90a970931d41fe --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hqq.py @@ -0,0 +1,129 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"HQQ (Half-Quadratic Quantization) integration file" + +from ..utils import is_hqq_available, is_torch_available, logging + + +if is_torch_available(): + import torch + +logger = logging.get_logger(__name__) + + +# Name all modules inside the model +def autoname_modules(model): + for name, module in model.named_modules(): + module.name = name + + +# Get the linear_tag from a modul name. For example: model.layers.31.self_attn.k_proj -> self_attn.k_proj +def name_to_linear_tag(name): + return ".".join([n for n in name.split(".") if ((n not in ["model", "layers"]) and (not n.isnumeric()))]) + + +# Get all linear tags available +def get_linear_tags(model): + if is_hqq_available(): + from hqq.core.quantize import HQQLinear + + linear_tags = set() + for name, module in model.named_modules(): + if isinstance(module, (torch.nn.Linear, HQQLinear)): + linear_tags.add(name_to_linear_tag(name)) + return list(linear_tags) + + +def _prepare_for_hqq_linear(model, patch_params, has_been_replaced, current_key_name=None): + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + + if isinstance(module, torch.nn.Linear): + # Get linear tag + linear_tag = name_to_linear_tag(module.name) + + # We put the module quant_config into the nn.Linear layer so we can access it later in quantizer_hqq.create_quantized_param() + if linear_tag in patch_params: + if patch_params[linear_tag] is not None: + model._modules[name].quant_config = patch_params[linear_tag] + # Store the module class in case we need to transpose the weight later + model._modules[name].source_cls = type(module) + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) + + has_been_replaced = True + + # Add these fake parameters to avoid loading fail + for att in ["W_q", "meta"]: + setattr(module, att, None) + + if len(list(module.children())) > 0: + _, has_been_replaced = _prepare_for_hqq_linear( + module, + patch_params=patch_params, + has_been_replaced=has_been_replaced, + ) + # Remove the last key for recursion + current_key_name.pop(-1) + + return model, has_been_replaced + + +def prepare_for_hqq_linear(model, quantization_config=None, modules_to_not_convert=None, has_been_replaced=False): + """ + Prepares nn.Linear layers for HQQ quantization. + Since each layer type can have separate quantization parameters, we need to do the following: + 1- tag each module with its neme via autoname_modules() + 2- Extract linear_tags (e.g. ['self_attn.q_proj', ...]) + 3- Map quantization parameters as a dictionary linear_tag -> quant_params as HQQLinear exepects it, this is referred to as patch_params + """ + + modules_to_not_convert = [] if modules_to_not_convert is None else modules_to_not_convert + + # Add name to module + autoname_modules(model) + + # Get linear tags. This allows us to use different quant params to different layer types + linear_tags = get_linear_tags(model) + + # Convert quantization_config to layer-wise config + skip_modules = quantization_config.skip_modules + quant_config = quantization_config.quant_config + linear_tags = list(set(linear_tags) - set(skip_modules) - set(modules_to_not_convert)) + + if any(key in linear_tags for key in quant_config.keys()): + # If the user doesn't specify a key from get_linear_tags, the layer is not quantized via (key, None) + patch_params = dict.fromkeys(linear_tags) + patch_params.update(quant_config) + else: + # Same quant_config for all layers + patch_params = dict.fromkeys(linear_tags, quant_config) + + model, has_been_replaced = _prepare_for_hqq_linear( + model, patch_params=patch_params, has_been_replaced=has_been_replaced + ) + + # We store quantization config as linear_tag -> hqq quant config + model.config.quantization_config = { + "quant_config": quant_config, + "quant_method": quantization_config.quant_method, + "skip_modules": skip_modules, + } + + if not has_been_replaced: + logger.warning("No linear modules were found in your model for quantization.") + + return model diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hub.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..57a267b72889d35074a71352d0630dfcefc9f747 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hub.py @@ -0,0 +1,1193 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Hub utilities: utilities related to download and cache models +""" + +import json +import os +import re +import sys +import tempfile +import warnings +from concurrent import futures +from pathlib import Path +from typing import Optional, Union +from urllib.parse import urlparse +from uuid import uuid4 + +import huggingface_hub +import requests +from huggingface_hub import ( + _CACHED_NO_EXIST, + CommitOperationAdd, + ModelCard, + ModelCardData, + constants, + create_branch, + create_commit, + create_repo, + hf_hub_download, + hf_hub_url, + snapshot_download, + try_to_load_from_cache, +) +from huggingface_hub.file_download import REGEX_COMMIT_HASH, http_get +from huggingface_hub.utils import ( + EntryNotFoundError, + GatedRepoError, + HfHubHTTPError, + LocalEntryNotFoundError, + OfflineModeIsEnabled, + RepositoryNotFoundError, + RevisionNotFoundError, + build_hf_headers, + get_session, + hf_raise_for_status, + send_telemetry, +) +from requests.exceptions import HTTPError + +from . import __version__, logging +from .generic import working_or_temp_dir +from .import_utils import ( + ENV_VARS_TRUE_VALUES, + _tf_version, + _torch_version, + is_tf_available, + is_torch_available, + is_training_run_on_sagemaker, +) + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +_is_offline_mode = huggingface_hub.constants.HF_HUB_OFFLINE + + +def is_offline_mode(): + return _is_offline_mode + + +torch_cache_home = os.getenv("TORCH_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "torch")) +default_cache_path = constants.default_cache_path + +# Determine default cache directory. Lots of legacy environment variables to ensure backward compatibility. +# The best way to set the cache path is with the environment variable HF_HOME. For more details, checkout this +# documentation page: https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables. +# +# In code, use `HF_HUB_CACHE` as the default cache path. This variable is set by the library and is guaranteed +# to be set to the right value. +# +# TODO: clean this for v5? +PYTORCH_PRETRAINED_BERT_CACHE = os.getenv("PYTORCH_PRETRAINED_BERT_CACHE", constants.HF_HUB_CACHE) +PYTORCH_TRANSFORMERS_CACHE = os.getenv("PYTORCH_TRANSFORMERS_CACHE", PYTORCH_PRETRAINED_BERT_CACHE) +TRANSFORMERS_CACHE = os.getenv("TRANSFORMERS_CACHE", PYTORCH_TRANSFORMERS_CACHE) + +HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(constants.HF_HOME, "modules")) +TRANSFORMERS_DYNAMIC_MODULE_NAME = "transformers_modules" +SESSION_ID = uuid4().hex + +# Add deprecation warning for old environment variables. +for key in ("PYTORCH_PRETRAINED_BERT_CACHE", "PYTORCH_TRANSFORMERS_CACHE", "TRANSFORMERS_CACHE"): + if os.getenv(key) is not None: + warnings.warn( + f"Using `{key}` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.", + FutureWarning, + ) + + +S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" +CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co" + +_staging_mode = os.environ.get("HUGGINGFACE_CO_STAGING", "NO").upper() in ENV_VARS_TRUE_VALUES +_default_endpoint = "https://hub-ci.huggingface.co" if _staging_mode else "https://huggingface.co" + +HUGGINGFACE_CO_RESOLVE_ENDPOINT = _default_endpoint +if os.environ.get("HUGGINGFACE_CO_RESOLVE_ENDPOINT", None) is not None: + warnings.warn( + "Using the environment variable `HUGGINGFACE_CO_RESOLVE_ENDPOINT` is deprecated and will be removed in " + "Transformers v5. Use `HF_ENDPOINT` instead.", + FutureWarning, + ) + HUGGINGFACE_CO_RESOLVE_ENDPOINT = os.environ.get("HUGGINGFACE_CO_RESOLVE_ENDPOINT", None) +HUGGINGFACE_CO_RESOLVE_ENDPOINT = os.environ.get("HF_ENDPOINT", HUGGINGFACE_CO_RESOLVE_ENDPOINT) +HUGGINGFACE_CO_PREFIX = HUGGINGFACE_CO_RESOLVE_ENDPOINT + "/{model_id}/resolve/{revision}/{filename}" +HUGGINGFACE_CO_EXAMPLES_TELEMETRY = HUGGINGFACE_CO_RESOLVE_ENDPOINT + "/api/telemetry/examples" + + +def _get_cache_file_to_return( + path_or_repo_id: str, full_filename: str, cache_dir: Union[str, Path, None] = None, revision: Optional[str] = None +): + # We try to see if we have a cached version (not up to date): + resolved_file = try_to_load_from_cache(path_or_repo_id, full_filename, cache_dir=cache_dir, revision=revision) + if resolved_file is not None and resolved_file != _CACHED_NO_EXIST: + return resolved_file + return None + + +def is_remote_url(url_or_filename): + parsed = urlparse(url_or_filename) + return parsed.scheme in ("http", "https") + + +def define_sagemaker_information(): + try: + instance_data = requests.get(os.environ["ECS_CONTAINER_METADATA_URI"]).json() + dlc_container_used = instance_data["Image"] + dlc_tag = instance_data["Image"].split(":")[1] + except Exception: + dlc_container_used = None + dlc_tag = None + + sagemaker_params = json.loads(os.getenv("SM_FRAMEWORK_PARAMS", "{}")) + runs_distributed_training = True if "sagemaker_distributed_dataparallel_enabled" in sagemaker_params else False + account_id = os.getenv("TRAINING_JOB_ARN").split(":")[4] if "TRAINING_JOB_ARN" in os.environ else None + + sagemaker_object = { + "sm_framework": os.getenv("SM_FRAMEWORK_MODULE", None), + "sm_region": os.getenv("AWS_REGION", None), + "sm_number_gpu": os.getenv("SM_NUM_GPUS", 0), + "sm_number_cpu": os.getenv("SM_NUM_CPUS", 0), + "sm_distributed_training": runs_distributed_training, + "sm_deep_learning_container": dlc_container_used, + "sm_deep_learning_container_tag": dlc_tag, + "sm_account_id": account_id, + } + return sagemaker_object + + +def http_user_agent(user_agent: Union[dict, str, None] = None) -> str: + """ + Formats a user-agent string with basic info about a request. + """ + ua = f"transformers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}" + if is_torch_available(): + ua += f"; torch/{_torch_version}" + if is_tf_available(): + ua += f"; tensorflow/{_tf_version}" + if constants.HF_HUB_DISABLE_TELEMETRY: + return ua + "; telemetry/off" + if is_training_run_on_sagemaker(): + ua += "; " + "; ".join(f"{k}/{v}" for k, v in define_sagemaker_information().items()) + # CI will set this value to True + if os.environ.get("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES: + ua += "; is_ci/true" + if isinstance(user_agent, dict): + ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items()) + elif isinstance(user_agent, str): + ua += "; " + user_agent + return ua + + +def extract_commit_hash(resolved_file: Optional[str], commit_hash: Optional[str]) -> Optional[str]: + """ + Extracts the commit hash from a resolved filename toward a cache file. + """ + if resolved_file is None or commit_hash is not None: + return commit_hash + resolved_file = str(Path(resolved_file).as_posix()) + search = re.search(r"snapshots/([^/]+)/", resolved_file) + if search is None: + return None + commit_hash = search.groups()[0] + return commit_hash if REGEX_COMMIT_HASH.match(commit_hash) else None + + +def cached_file( + path_or_repo_id: Union[str, os.PathLike], + filename: str, + **kwargs, +) -> Optional[str]: + """ + Tries to locate a file in a local folder and repo, downloads and cache it if necessary. + + Args: + path_or_repo_id (`str` or `os.PathLike`): + This can be either: + - a string, the *model id* of a model repo on huggingface.co. + - a path to a *directory* potentially containing the file. + filename (`str`): + The name of the file to locate in `path_or_repo`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can + specify the folder name here. + repo_type (`str`, *optional*): + Specify the repo type (useful when downloading from a space for instance). + + + + Passing `token=True` is required when you want to use a private model. + + + + Returns: + `Optional[str]`: Returns the resolved file (to the cache folder if downloaded from a repo). + + Examples: + + ```python + # Download a model weight from the Hub and cache it. + model_weights_file = cached_file("google-bert/bert-base-uncased", "pytorch_model.bin") + ``` + """ + file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs) + file = file[0] if file is not None else file + return file + + +def cached_files( + path_or_repo_id: Union[str, os.PathLike], + filenames: list[str], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + resume_download: Optional[bool] = None, + proxies: Optional[dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + local_files_only: bool = False, + subfolder: str = "", + repo_type: Optional[str] = None, + user_agent: Optional[Union[str, dict[str, str]]] = None, + _raise_exceptions_for_gated_repo: bool = True, + _raise_exceptions_for_missing_entries: bool = True, + _raise_exceptions_for_connection_errors: bool = True, + _commit_hash: Optional[str] = None, + **deprecated_kwargs, +) -> Optional[str]: + """ + Tries to locate several files in a local folder and repo, downloads and cache them if necessary. + + Args: + path_or_repo_id (`str` or `os.PathLike`): + This can be either: + - a string, the *model id* of a model repo on huggingface.co. + - a path to a *directory* potentially containing the file. + filenames (`List[str]`): + The name of all the files to locate in `path_or_repo`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can + specify the folder name here. + repo_type (`str`, *optional*): + Specify the repo type (useful when downloading from a space for instance). + + Private args: + _raise_exceptions_for_gated_repo (`bool`): + if False, do not raise an exception for gated repo error but return None. + _raise_exceptions_for_missing_entries (`bool`): + if False, do not raise an exception for missing entries but return None. + _raise_exceptions_for_connection_errors (`bool`): + if False, do not raise an exception for connection errors but return None. + _commit_hash (`str`, *optional*): + passed when we are chaining several calls to various files (e.g. when loading a tokenizer or + a pipeline). If files are cached for this commit hash, avoid calls to head and get from the cache. + + + + Passing `token=True` is required when you want to use a private model. + + + + Returns: + `Optional[str]`: Returns the resolved file (to the cache folder if downloaded from a repo). + + Examples: + + ```python + # Download a model weight from the Hub and cache it. + model_weights_file = cached_file("google-bert/bert-base-uncased", "pytorch_model.bin") + ``` + """ + use_auth_token = deprecated_kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + token = use_auth_token + + if is_offline_mode() and not local_files_only: + logger.info("Offline mode: forcing local_files_only=True") + local_files_only = True + if subfolder is None: + subfolder = "" + + # Add folder to filenames + full_filenames = [os.path.join(subfolder, file) for file in filenames] + + path_or_repo_id = str(path_or_repo_id) + existing_files = [] + for filename in full_filenames: + if os.path.isdir(path_or_repo_id): + resolved_file = os.path.join(path_or_repo_id, filename) + if not os.path.isfile(resolved_file): + if _raise_exceptions_for_missing_entries and filename != os.path.join(subfolder, "config.json"): + revision_ = "main" if revision is None else revision + raise OSError( + f"{path_or_repo_id} does not appear to have a file named {filename}. Checkout " + f"'https://huggingface.co/{path_or_repo_id}/tree/{revision_}' for available files." + ) + else: + return None + existing_files.append(resolved_file) + + # All files exist + if len(existing_files) == len(full_filenames): + return existing_files + + if cache_dir is None: + cache_dir = TRANSFORMERS_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + existing_files = [] + file_counter = 0 + if _commit_hash is not None and not force_download: + for filename in full_filenames: + # If the file is cached under that commit hash, we return it directly. + resolved_file = try_to_load_from_cache( + path_or_repo_id, filename, cache_dir=cache_dir, revision=_commit_hash, repo_type=repo_type + ) + if resolved_file is not None: + if resolved_file is not _CACHED_NO_EXIST: + file_counter += 1 + existing_files.append(resolved_file) + elif not _raise_exceptions_for_missing_entries: + file_counter += 1 + else: + raise OSError(f"Could not locate {filename} inside {path_or_repo_id}.") + + # Either all the files were found, or some were _CACHED_NO_EXIST but we do not raise for missing entries + if file_counter == len(full_filenames): + return existing_files if len(existing_files) > 0 else None + + user_agent = http_user_agent(user_agent) + # download the files if needed + try: + if len(full_filenames) == 1: + # This is slightly better for only 1 file + hf_hub_download( + path_or_repo_id, + filenames[0], + subfolder=None if len(subfolder) == 0 else subfolder, + repo_type=repo_type, + revision=revision, + cache_dir=cache_dir, + user_agent=user_agent, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + token=token, + local_files_only=local_files_only, + ) + else: + snapshot_download( + path_or_repo_id, + allow_patterns=full_filenames, + repo_type=repo_type, + revision=revision, + cache_dir=cache_dir, + user_agent=user_agent, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + token=token, + local_files_only=local_files_only, + ) + + except Exception as e: + # We cannot recover from them + if isinstance(e, RepositoryNotFoundError) and not isinstance(e, GatedRepoError): + raise OSError( + f"{path_or_repo_id} is not a local folder and is not a valid model identifier " + "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a token " + "having permission to this repo either by logging in with `huggingface-cli login` or by passing " + "`token=`" + ) from e + elif isinstance(e, RevisionNotFoundError): + raise OSError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists " + "for this model name. Check the model page at " + f"'https://huggingface.co/{path_or_repo_id}' for available revisions." + ) from e + + # Now we try to recover if we can find all files correctly in the cache + resolved_files = [ + _get_cache_file_to_return(path_or_repo_id, filename, cache_dir, revision) for filename in full_filenames + ] + if all(file is not None for file in resolved_files): + return resolved_files + + # Raise based on the flags. Note that we will raise for missing entries at the very end, even when + # not entering this Except block, as it may also happen when `snapshot_download` does not raise + if isinstance(e, GatedRepoError): + if not _raise_exceptions_for_gated_repo: + return None + raise OSError( + "You are trying to access a gated repo.\nMake sure to have access to it at " + f"https://huggingface.co/{path_or_repo_id}.\n{str(e)}" + ) from e + elif isinstance(e, LocalEntryNotFoundError): + if not _raise_exceptions_for_connection_errors: + return None + # Here we only raise if both flags for missing entry and connection errors are True (because it can be raised + # even when `local_files_only` is True, in which case raising for connections errors only would not make sense) + elif _raise_exceptions_for_missing_entries: + raise OSError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load the files, and couldn't find them in the" + f" cached files.\nCheckout your internet connection or see how to run the library in offline mode at" + " 'https://huggingface.co/docs/transformers/installation#offline-mode'." + ) from e + # snapshot_download will not raise EntryNotFoundError, but hf_hub_download can. If this is the case, it will be treated + # later on anyway and re-raised if needed + elif isinstance(e, HTTPError) and not isinstance(e, EntryNotFoundError): + if not _raise_exceptions_for_connection_errors: + return None + raise OSError(f"There was a specific connection error when trying to load {path_or_repo_id}:\n{e}") + + resolved_files = [ + _get_cache_file_to_return(path_or_repo_id, filename, cache_dir, revision) for filename in full_filenames + ] + # If there are any missing file and the flag is active, raise + if any(file is None for file in resolved_files) and _raise_exceptions_for_missing_entries: + missing_entries = [original for original, resolved in zip(full_filenames, resolved_files) if resolved is None] + # Last escape + if len(resolved_files) == 1 and missing_entries[0] == os.path.join(subfolder, "config.json"): + return None + # Now we raise for missing entries + revision_ = "main" if revision is None else revision + msg = ( + f"a file named {missing_entries[0]}" if len(missing_entries) == 1 else f"files named {(*missing_entries,)}" + ) + raise EnvironmentError( + f"{path_or_repo_id} does not appear to have {msg}. Checkout 'https://huggingface.co/{path_or_repo_id}/tree/{revision_}'" + "for available files." + ) + + # Remove potential missing entries (we can silently remove them at this point based on the flags) + resolved_files = [file for file in resolved_files if file is not None] + # Return `None` if the list is empty, coherent with other Exception when the flag is not active + resolved_files = None if len(resolved_files) == 0 else resolved_files + + return resolved_files + + +# TODO cyril: Deprecated and should be removed in 4.51 +def get_file_from_repo( + *args, + **kwargs, +): + """ + Tries to locate a file in a local folder and repo, downloads and cache it if necessary. + + Args: + path_or_repo (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a model repo on huggingface.co. + - a path to a *directory* potentially containing the file. + filename (`str`): + The name of the file to locate in `path_or_repo`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can + specify the folder name here. + + + + Passing `token=True` is required when you want to use a private model. + + + + Returns: + `Optional[str]`: Returns the resolved file (to the cache folder if downloaded from a repo) or `None` if the + file does not exist. + + Examples: + + ```python + # Download a tokenizer configuration from huggingface.co and cache. + tokenizer_config = get_file_from_repo("google-bert/bert-base-uncased", "tokenizer_config.json") + # This model does not have a tokenizer config so the result will be None. + tokenizer_config = get_file_from_repo("FacebookAI/xlm-roberta-base", "tokenizer_config.json") + ``` + """ + logger.warning( + "`get_file_from_repo` is deprecated and will be removed in version 4.51. Use `cached_file` instead." + ) + return cached_file( + *args, + _raise_exceptions_for_gated_repo=False, + _raise_exceptions_for_missing_entries=False, + _raise_exceptions_for_connection_errors=False, + **kwargs, + ) + + +def download_url(url, proxies=None): + """ + Downloads a given url in a temporary file. This function is not safe to use in multiple processes. Its only use is + for deprecated behavior allowing to download config/models with a single url instead of using the Hub. + + Args: + url (`str`): The url of the file to download. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + + Returns: + `str`: The location of the temporary file where the url was downloaded. + """ + warnings.warn( + f"Using `from_pretrained` with the url of a file (here {url}) is deprecated and won't be possible anymore in" + " v5 of Transformers. You should host your file on the Hub (hf.co) instead and use the repository ID. Note" + " that this is not compatible with the caching system (your file will be downloaded at each execution) or" + " multiple processes (each process will download the file in a different temporary file).", + FutureWarning, + ) + tmp_fd, tmp_file = tempfile.mkstemp() + with os.fdopen(tmp_fd, "wb") as f: + http_get(url, f, proxies=proxies) + return tmp_file + + +def has_file( + path_or_repo: Union[str, os.PathLike], + filename: str, + revision: Optional[str] = None, + proxies: Optional[dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + *, + local_files_only: bool = False, + cache_dir: Union[str, Path, None] = None, + repo_type: Optional[str] = None, + **deprecated_kwargs, +): + """ + Checks if a repo contains a given file without downloading it. Works for remote repos and local folders. + + If offline mode is enabled, checks if the file exists in the cache. + + + + This function will raise an error if the repository `path_or_repo` is not valid or if `revision` does not exist for + this repo, but will return False for regular connection errors. + + + """ + use_auth_token = deprecated_kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + token = use_auth_token + + # If path to local directory, check if the file exists + if os.path.isdir(path_or_repo): + return os.path.isfile(os.path.join(path_or_repo, filename)) + + # Else it's a repo => let's check if the file exists in local cache or on the Hub + + # Check if file exists in cache + # This information might be outdated so it's best to also make a HEAD call (if allowed). + cached_path = try_to_load_from_cache( + repo_id=path_or_repo, + filename=filename, + revision=revision, + repo_type=repo_type, + cache_dir=cache_dir, + ) + has_file_in_cache = isinstance(cached_path, str) + + # If local_files_only, don't try the HEAD call + if local_files_only: + return has_file_in_cache + + # Check if the file exists + try: + response = get_session().head( + hf_hub_url(path_or_repo, filename=filename, revision=revision, repo_type=repo_type), + headers=build_hf_headers(token=token, user_agent=http_user_agent()), + allow_redirects=False, + proxies=proxies, + timeout=10, + ) + except (requests.exceptions.SSLError, requests.exceptions.ProxyError): + # Actually raise for those subclasses of ConnectionError + raise + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + OfflineModeIsEnabled, + ): + return has_file_in_cache + + try: + hf_raise_for_status(response) + return True + except GatedRepoError as e: + logger.error(e) + raise OSError( + f"{path_or_repo} is a gated repository. Make sure to request access at " + f"https://huggingface.co/{path_or_repo} and pass a token having permission to this repo either by " + "logging in with `huggingface-cli login` or by passing `token=`." + ) from e + except RepositoryNotFoundError as e: + logger.error(e) + raise OSError(f"{path_or_repo} is not a local folder or a valid repository name on 'https://hf.co'.") from e + except RevisionNotFoundError as e: + logger.error(e) + raise OSError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for this " + f"model name. Check the model page at 'https://huggingface.co/{path_or_repo}' for available revisions." + ) from e + except EntryNotFoundError: + return False # File does not exist + except requests.HTTPError: + # Any authentication/authorization error will be caught here => default to cache + return has_file_in_cache + + +class PushToHubMixin: + """ + A Mixin containing the functionality to push a model or tokenizer to the hub. + """ + + def _create_repo( + self, + repo_id: str, + private: Optional[bool] = None, + token: Optional[Union[bool, str]] = None, + repo_url: Optional[str] = None, + organization: Optional[str] = None, + ) -> str: + """ + Create the repo if needed, cleans up repo_id with deprecated kwargs `repo_url` and `organization`, retrieves + the token. + """ + if repo_url is not None: + warnings.warn( + "The `repo_url` argument is deprecated and will be removed in v5 of Transformers. Use `repo_id` " + "instead." + ) + if repo_id is not None: + raise ValueError( + "`repo_id` and `repo_url` are both specified. Please set only the argument `repo_id`." + ) + repo_id = repo_url.replace(f"{HUGGINGFACE_CO_RESOLVE_ENDPOINT}/", "") + if organization is not None: + warnings.warn( + "The `organization` argument is deprecated and will be removed in v5 of Transformers. Set your " + "organization directly in the `repo_id` passed instead (`repo_id={organization}/{model_id}`)." + ) + if not repo_id.startswith(organization): + if "/" in repo_id: + repo_id = repo_id.split("/")[-1] + repo_id = f"{organization}/{repo_id}" + + url = create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True) + return url.repo_id + + def _get_files_timestamps(self, working_dir: Union[str, os.PathLike]): + """ + Returns the list of files with their last modification timestamp. + """ + return {f: os.path.getmtime(os.path.join(working_dir, f)) for f in os.listdir(working_dir)} + + def _upload_modified_files( + self, + working_dir: Union[str, os.PathLike], + repo_id: str, + files_timestamps: dict[str, float], + commit_message: Optional[str] = None, + token: Optional[Union[bool, str]] = None, + create_pr: bool = False, + revision: Optional[str] = None, + commit_description: Optional[str] = None, + ): + """ + Uploads all modified files in `working_dir` to `repo_id`, based on `files_timestamps`. + """ + if commit_message is None: + if "Model" in self.__class__.__name__: + commit_message = "Upload model" + elif "Config" in self.__class__.__name__: + commit_message = "Upload config" + elif "Tokenizer" in self.__class__.__name__: + commit_message = "Upload tokenizer" + elif "FeatureExtractor" in self.__class__.__name__: + commit_message = "Upload feature extractor" + elif "Processor" in self.__class__.__name__: + commit_message = "Upload processor" + else: + commit_message = f"Upload {self.__class__.__name__}" + modified_files = [ + f + for f in os.listdir(working_dir) + if f not in files_timestamps or os.path.getmtime(os.path.join(working_dir, f)) > files_timestamps[f] + ] + + # filter for actual files + folders at the root level + modified_files = [ + f + for f in modified_files + if os.path.isfile(os.path.join(working_dir, f)) or os.path.isdir(os.path.join(working_dir, f)) + ] + + operations = [] + # upload standalone files + for file in modified_files: + if os.path.isdir(os.path.join(working_dir, file)): + # go over individual files of folder + for f in os.listdir(os.path.join(working_dir, file)): + operations.append( + CommitOperationAdd( + path_or_fileobj=os.path.join(working_dir, file, f), path_in_repo=os.path.join(file, f) + ) + ) + else: + operations.append( + CommitOperationAdd(path_or_fileobj=os.path.join(working_dir, file), path_in_repo=file) + ) + + if revision is not None and not revision.startswith("refs/pr"): + try: + create_branch(repo_id=repo_id, branch=revision, token=token, exist_ok=True) + except HfHubHTTPError as e: + if e.response.status_code == 403 and create_pr: + # If we are creating a PR on a repo we don't have access to, we can't create the branch. + # so let's assume the branch already exists. If it's not the case, an error will be raised when + # calling `create_commit` below. + pass + else: + raise + + logger.info(f"Uploading the following files to {repo_id}: {','.join(modified_files)}") + return create_commit( + repo_id=repo_id, + operations=operations, + commit_message=commit_message, + commit_description=commit_description, + token=token, + create_pr=create_pr, + revision=revision, + ) + + def push_to_hub( + self, + repo_id: str, + use_temp_dir: Optional[bool] = None, + commit_message: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[Union[bool, str]] = None, + max_shard_size: Optional[Union[int, str]] = "5GB", + create_pr: bool = False, + safe_serialization: bool = True, + revision: Optional[str] = None, + commit_description: Optional[str] = None, + tags: Optional[list[str]] = None, + **deprecated_kwargs, + ) -> str: + """ + Upload the {object_files} to the 🤗 Model Hub. + + Parameters: + repo_id (`str`): + The name of the repository you want to push your {object} to. It should contain your organization name + when pushing to a given organization. + use_temp_dir (`bool`, *optional*): + Whether or not to use a temporary directory to store the files saved before they are pushed to the Hub. + Will default to `True` if there is no directory named like `repo_id`, `False` otherwise. + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to `"Upload {object}"`. + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. + token (`bool` or `str`, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). Will default to `True` if `repo_url` + is not specified. + max_shard_size (`int` or `str`, *optional*, defaults to `"5GB"`): + Only applicable for models. The maximum size for a checkpoint before being sharded. Checkpoints shard + will then be each of size lower than this size. If expressed as a string, needs to be digits followed + by a unit (like `"5MB"`). We default it to `"5GB"` so that users can easily load models on free-tier + Google Colab instances without any CPU OOM issues. + create_pr (`bool`, *optional*, defaults to `False`): + Whether or not to create a PR with the uploaded files or directly commit. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether or not to convert the model weights in safetensors format for safer serialization. + revision (`str`, *optional*): + Branch to push the uploaded files to. + commit_description (`str`, *optional*): + The description of the commit that will be created + tags (`List[str]`, *optional*): + List of tags to push on the Hub. + + Examples: + + ```python + from transformers import {object_class} + + {object} = {object_class}.from_pretrained("google-bert/bert-base-cased") + + # Push the {object} to your namespace with the name "my-finetuned-bert". + {object}.push_to_hub("my-finetuned-bert") + + # Push the {object} to an organization with the name "my-finetuned-bert". + {object}.push_to_hub("huggingface/my-finetuned-bert") + ``` + """ + use_auth_token = deprecated_kwargs.pop("use_auth_token", None) + ignore_metadata_errors = deprecated_kwargs.pop("ignore_metadata_errors", False) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + repo_path_or_name = deprecated_kwargs.pop("repo_path_or_name", None) + if repo_path_or_name is not None: + # Should use `repo_id` instead of `repo_path_or_name`. When using `repo_path_or_name`, we try to infer + # repo_id from the folder path, if it exists. + warnings.warn( + "The `repo_path_or_name` argument is deprecated and will be removed in v5 of Transformers. Use " + "`repo_id` instead.", + FutureWarning, + ) + if repo_id is not None: + raise ValueError( + "`repo_id` and `repo_path_or_name` are both specified. Please set only the argument `repo_id`." + ) + if os.path.isdir(repo_path_or_name): + # repo_path: infer repo_id from the path + repo_id = repo_id.split(os.path.sep)[-1] + working_dir = repo_id + else: + # repo_name: use it as repo_id + repo_id = repo_path_or_name + working_dir = repo_id.split("/")[-1] + else: + # Repo_id is passed correctly: infer working_dir from it + working_dir = repo_id.split("/")[-1] + + # Deprecation warning will be sent after for repo_url and organization + repo_url = deprecated_kwargs.pop("repo_url", None) + organization = deprecated_kwargs.pop("organization", None) + + repo_id = self._create_repo( + repo_id, private=private, token=token, repo_url=repo_url, organization=organization + ) + + # Create a new empty model card and eventually tag it + model_card = create_and_tag_model_card( + repo_id, tags, token=token, ignore_metadata_errors=ignore_metadata_errors + ) + + if use_temp_dir is None: + use_temp_dir = not os.path.isdir(working_dir) + + with working_or_temp_dir(working_dir=working_dir, use_temp_dir=use_temp_dir) as work_dir: + files_timestamps = self._get_files_timestamps(work_dir) + + # Save all files. + self.save_pretrained(work_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization) + + # Update model card if needed: + model_card.save(os.path.join(work_dir, "README.md")) + + return self._upload_modified_files( + work_dir, + repo_id, + files_timestamps, + commit_message=commit_message, + token=token, + create_pr=create_pr, + revision=revision, + commit_description=commit_description, + ) + + +def send_example_telemetry(example_name, *example_args, framework="pytorch"): + """ + Sends telemetry that helps tracking the examples use. + + Args: + example_name (`str`): The name of the example. + *example_args (dataclasses or `argparse.ArgumentParser`): The arguments to the script. This function will only + try to extract the model and dataset name from those. Nothing else is tracked. + framework (`str`, *optional*, defaults to `"pytorch"`): The framework for the example. + """ + if is_offline_mode(): + return + + data = {"example": example_name, "framework": framework} + for args in example_args: + args_as_dict = {k: v for k, v in args.__dict__.items() if not k.startswith("_") and v is not None} + if "model_name_or_path" in args_as_dict: + model_name = args_as_dict["model_name_or_path"] + # Filter out local paths + if not os.path.isdir(model_name): + data["model_name"] = args_as_dict["model_name_or_path"] + if "dataset_name" in args_as_dict: + data["dataset_name"] = args_as_dict["dataset_name"] + elif "task_name" in args_as_dict: + # Extract script name from the example_name + script_name = example_name.replace("tf_", "").replace("flax_", "").replace("run_", "") + script_name = script_name.replace("_no_trainer", "") + data["dataset_name"] = f"{script_name}-{args_as_dict['task_name']}" + + # Send telemetry in the background + send_telemetry( + topic="examples", library_name="transformers", library_version=__version__, user_agent=http_user_agent(data) + ) + + +def convert_file_size_to_int(size: Union[int, str]): + """ + Converts a size expressed as a string with digits an unit (like `"5MB"`) to an integer (in bytes). + + Args: + size (`int` or `str`): The size to convert. Will be directly returned if an `int`. + + Example: + ```py + >>> convert_file_size_to_int("1MiB") + 1048576 + ``` + """ + if isinstance(size, int): + return size + if size.upper().endswith("GIB"): + return int(size[:-3]) * (2**30) + if size.upper().endswith("MIB"): + return int(size[:-3]) * (2**20) + if size.upper().endswith("KIB"): + return int(size[:-3]) * (2**10) + if size.upper().endswith("GB"): + int_size = int(size[:-2]) * (10**9) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("MB"): + int_size = int(size[:-2]) * (10**6) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("KB"): + int_size = int(size[:-2]) * (10**3) + return int_size // 8 if size.endswith("b") else int_size + raise ValueError("`size` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.") + + +def get_checkpoint_shard_files( + pretrained_model_name_or_path, + index_filename, + cache_dir=None, + force_download=False, + proxies=None, + resume_download=None, + local_files_only=False, + token=None, + user_agent=None, + revision=None, + subfolder="", + _commit_hash=None, + **deprecated_kwargs, +): + """ + For a given model: + + - download and cache all the shards of a sharded checkpoint if `pretrained_model_name_or_path` is a model ID on the + Hub + - returns the list of paths to all the shards, as well as some metadata. + + For the description of each arg, see [`PreTrainedModel.from_pretrained`]. `index_filename` is the full path to the + index (downloaded and cached if `pretrained_model_name_or_path` is a model ID on the Hub). + """ + import json + + use_auth_token = deprecated_kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + token = use_auth_token + + if not os.path.isfile(index_filename): + raise ValueError(f"Can't find a checkpoint index ({index_filename}) in {pretrained_model_name_or_path}.") + + with open(index_filename) as f: + index = json.loads(f.read()) + + shard_filenames = sorted(set(index["weight_map"].values())) + sharded_metadata = index["metadata"] + sharded_metadata["all_checkpoint_keys"] = list(index["weight_map"].keys()) + sharded_metadata["weight_map"] = index["weight_map"].copy() + + # First, let's deal with local folder. + if os.path.isdir(pretrained_model_name_or_path): + shard_filenames = [os.path.join(pretrained_model_name_or_path, subfolder, f) for f in shard_filenames] + return shard_filenames, sharded_metadata + + # At this stage pretrained_model_name_or_path is a model identifier on the Hub. Try to get everything from cache, + # or download the files + cached_filenames = cached_files( + pretrained_model_name_or_path, + shard_filenames, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + revision=revision, + subfolder=subfolder, + _commit_hash=_commit_hash, + ) + + return cached_filenames, sharded_metadata + + +def create_and_tag_model_card( + repo_id: str, + tags: Optional[list[str]] = None, + token: Optional[str] = None, + ignore_metadata_errors: bool = False, +): + """ + Creates or loads an existing model card and tags it. + + Args: + repo_id (`str`): + The repo_id where to look for the model card. + tags (`List[str]`, *optional*): + The list of tags to add in the model card + token (`str`, *optional*): + Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token. + ignore_metadata_errors (`bool`, *optional*, defaults to `False`): + If True, errors while parsing the metadata section will be ignored. Some information might be lost during + the process. Use it at your own risk. + """ + try: + # Check if the model card is present on the remote repo + model_card = ModelCard.load(repo_id, token=token, ignore_metadata_errors=ignore_metadata_errors) + except EntryNotFoundError: + # Otherwise create a simple model card from template + model_description = "This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated." + card_data = ModelCardData(tags=[] if tags is None else tags, library_name="transformers") + model_card = ModelCard.from_template(card_data, model_description=model_description) + + if tags is not None: + # Ensure model_card.data.tags is a list and not None + if model_card.data.tags is None: + model_card.data.tags = [] + for model_tag in tags: + if model_tag not in model_card.data.tags: + model_card.data.tags.append(model_tag) + + return model_card + + +class PushInProgress: + """ + Internal class to keep track of a push in progress (which might contain multiple `Future` jobs). + """ + + def __init__(self, jobs: Optional[futures.Future] = None) -> None: + self.jobs = [] if jobs is None else jobs + + def is_done(self): + return all(job.done() for job in self.jobs) + + def wait_until_done(self): + futures.wait(self.jobs) + + def cancel(self) -> None: + self.jobs = [ + job + for job in self.jobs + # Cancel the job if it wasn't started yet and remove cancelled/done jobs from the list + if not (job.cancel() or job.done()) + ] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hub_kernels.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hub_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..b2ec6b53715aca2068db6e18ac4753d2720b9b09 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hub_kernels.py @@ -0,0 +1,73 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Union + + +try: + from kernels import ( + Device, + LayerRepository, + register_kernel_mapping, + replace_kernel_forward_from_hub, + use_kernel_forward_from_hub, + ) + + _hub_kernels_available = True + + _KERNEL_MAPPING: Dict[str, Dict[Union[Device, str], LayerRepository]] = { + "MultiScaleDeformableAttention": { + "cuda": LayerRepository( + repo_id="kernels-community/deformable-detr", + layer_name="MultiScaleDeformableAttention", + ) + } + } + + register_kernel_mapping(_KERNEL_MAPPING) + +except ImportError: + # Stub to make decorators int transformers work when `kernels` + # is not installed. + def use_kernel_forward_from_hub(*args, **kwargs): + def decorator(cls): + return cls + + return decorator + + class LayerRepository: + def __init__(self, *args, **kwargs): + raise RuntimeError("LayerRepository requires `kernels` to be installed. Run `pip install kernels`.") + + def replace_kernel_forward_from_hub(*args, **kwargs): + raise RuntimeError( + "replace_kernel_forward_from_hub requires `kernels` to be installed. Run `pip install kernels`." + ) + + def register_kernel_mapping(*args, **kwargs): + raise RuntimeError("register_kernel_mapping requires `kernels` to be installed. Run `pip install kernels`.") + + _hub_kernels_available = False + + +def is_hub_kernels_available(): + return _hub_kernels_available + + +__all__ = [ + "LayerRepository", + "is_hub_kernels_available", + "use_kernel_forward_from_hub", + "register_kernel_mapping", + "replace_kernel_forward_from_hub", +] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hyperparameter_search.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hyperparameter_search.py new file mode 100644 index 0000000000000000000000000000000000000000..e8558ceed32f4640c727a192d3ab00ed9104986f --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/hyperparameter_search.py @@ -0,0 +1,141 @@ +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +from .integrations import ( + is_optuna_available, + is_ray_tune_available, + is_sigopt_available, + is_wandb_available, + run_hp_search_optuna, + run_hp_search_ray, + run_hp_search_sigopt, + run_hp_search_wandb, +) +from .trainer_utils import ( + HPSearchBackend, + default_hp_space_optuna, + default_hp_space_ray, + default_hp_space_sigopt, + default_hp_space_wandb, +) +from .utils import logging + + +logger = logging.get_logger(__name__) + + +class HyperParamSearchBackendBase: + name: str + pip_package: Optional[str] = None + + @staticmethod + def is_available(): + raise NotImplementedError + + def run(self, trainer, n_trials: int, direction: str, **kwargs): + raise NotImplementedError + + def default_hp_space(self, trial): + raise NotImplementedError + + def ensure_available(self): + if not self.is_available(): + raise RuntimeError( + f"You picked the {self.name} backend, but it is not installed. Run {self.pip_install()}." + ) + + @classmethod + def pip_install(cls): + return f"`pip install {cls.pip_package or cls.name}`" + + +class OptunaBackend(HyperParamSearchBackendBase): + name = "optuna" + + @staticmethod + def is_available(): + return is_optuna_available() + + def run(self, trainer, n_trials: int, direction: str, **kwargs): + return run_hp_search_optuna(trainer, n_trials, direction, **kwargs) + + def default_hp_space(self, trial): + return default_hp_space_optuna(trial) + + +class RayTuneBackend(HyperParamSearchBackendBase): + name = "ray" + pip_package = "'ray[tune]'" + + @staticmethod + def is_available(): + return is_ray_tune_available() + + def run(self, trainer, n_trials: int, direction: str, **kwargs): + return run_hp_search_ray(trainer, n_trials, direction, **kwargs) + + def default_hp_space(self, trial): + return default_hp_space_ray(trial) + + +class SigOptBackend(HyperParamSearchBackendBase): + name = "sigopt" + + @staticmethod + def is_available(): + return is_sigopt_available() + + def run(self, trainer, n_trials: int, direction: str, **kwargs): + return run_hp_search_sigopt(trainer, n_trials, direction, **kwargs) + + def default_hp_space(self, trial): + return default_hp_space_sigopt(trial) + + +class WandbBackend(HyperParamSearchBackendBase): + name = "wandb" + + @staticmethod + def is_available(): + return is_wandb_available() + + def run(self, trainer, n_trials: int, direction: str, **kwargs): + return run_hp_search_wandb(trainer, n_trials, direction, **kwargs) + + def default_hp_space(self, trial): + return default_hp_space_wandb(trial) + + +ALL_HYPERPARAMETER_SEARCH_BACKENDS = { + HPSearchBackend(backend.name): backend for backend in [OptunaBackend, RayTuneBackend, SigOptBackend, WandbBackend] +} + + +def default_hp_search_backend() -> str: + available_backends = [backend for backend in ALL_HYPERPARAMETER_SEARCH_BACKENDS.values() if backend.is_available()] + if len(available_backends) > 0: + name = available_backends[0].name + if len(available_backends) > 1: + logger.info( + f"{len(available_backends)} hyperparameter search backends available. Using {name} as the default." + ) + return name + raise RuntimeError( + "No hyperparameter search backend available.\n" + + "\n".join( + f" - To install {backend.name} run {backend.pip_install()}" + for backend in ALL_HYPERPARAMETER_SEARCH_BACKENDS.values() + ) + ) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_auto.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_auto.py new file mode 100644 index 0000000000000000000000000000000000000000..2f9d42fcdb7290b617d13e29c166a070b89f6db5 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_auto.py @@ -0,0 +1,642 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AutoImageProcessor class.""" + +import importlib +import json +import os +import warnings +from collections import OrderedDict +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union + +# Build the list of all image processors +from ...configuration_utils import PretrainedConfig +from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code +from ...image_processing_utils import ImageProcessingMixin +from ...image_processing_utils_fast import BaseImageProcessorFast +from ...utils import ( + CONFIG_NAME, + IMAGE_PROCESSOR_NAME, + cached_file, + is_timm_config_dict, + is_timm_local_checkpoint, + is_torchvision_available, + is_vision_available, + logging, +) +from .auto_factory import _LazyAutoMapping +from .configuration_auto import ( + CONFIG_MAPPING_NAMES, + AutoConfig, + model_type_to_module_name, + replace_list_option_in_docstrings, +) + + +logger = logging.get_logger(__name__) + + +if TYPE_CHECKING: + # This significantly improves completion suggestion performance when + # the transformers package is used with Microsoft's Pylance language server. + IMAGE_PROCESSOR_MAPPING_NAMES: OrderedDict[str, Tuple[Optional[str], Optional[str]]] = OrderedDict() +else: + IMAGE_PROCESSOR_MAPPING_NAMES = OrderedDict( + [ + ("align", ("EfficientNetImageProcessor",)), + ("aria", ("AriaImageProcessor",)), + ("beit", ("BeitImageProcessor",)), + ("bit", ("BitImageProcessor",)), + ("blip", ("BlipImageProcessor", "BlipImageProcessorFast")), + ("blip-2", ("BlipImageProcessor", "BlipImageProcessorFast")), + ("bridgetower", ("BridgeTowerImageProcessor",)), + ("chameleon", ("ChameleonImageProcessor",)), + ("chinese_clip", ("ChineseCLIPImageProcessor",)), + ("clip", ("CLIPImageProcessor", "CLIPImageProcessorFast")), + ("clipseg", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("conditional_detr", ("ConditionalDetrImageProcessor",)), + ("convnext", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), + ("convnextv2", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), + ("cvt", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), + ("data2vec-vision", ("BeitImageProcessor",)), + ("deformable_detr", ("DeformableDetrImageProcessor", "DeformableDetrImageProcessorFast")), + ("deit", ("DeiTImageProcessor", "DeiTImageProcessorFast")), + ("depth_anything", ("DPTImageProcessor",)), + ("depth_pro", ("DepthProImageProcessor", "DepthProImageProcessorFast")), + ("deta", ("DetaImageProcessor",)), + ("detr", ("DetrImageProcessor", "DetrImageProcessorFast")), + ("dinat", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("dinov2", ("BitImageProcessor",)), + ("donut-swin", ("DonutImageProcessor",)), + ("dpt", ("DPTImageProcessor",)), + ("efficientformer", ("EfficientFormerImageProcessor",)), + ("efficientnet", ("EfficientNetImageProcessor",)), + ("flava", ("FlavaImageProcessor",)), + ("focalnet", ("BitImageProcessor",)), + ("fuyu", ("FuyuImageProcessor",)), + ("gemma3", ("Gemma3ImageProcessor", "Gemma3ImageProcessorFast")), + ("git", ("CLIPImageProcessor", "CLIPImageProcessorFast")), + ("glpn", ("GLPNImageProcessor",)), + ("got_ocr2", ("GotOcr2ImageProcessor", "GotOcr2ImageProcessorFast")), + ("grounding-dino", ("GroundingDinoImageProcessor",)), + ("groupvit", ("CLIPImageProcessor", "CLIPImageProcessorFast")), + ("hiera", ("BitImageProcessor",)), + ("idefics", ("IdeficsImageProcessor",)), + ("idefics2", ("Idefics2ImageProcessor",)), + ("idefics3", ("Idefics3ImageProcessor",)), + ("ijepa", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("imagegpt", ("ImageGPTImageProcessor",)), + ("instructblip", ("BlipImageProcessor", "BlipImageProcessorFast")), + ("instructblipvideo", ("InstructBlipVideoImageProcessor",)), + ("kosmos-2", ("CLIPImageProcessor", "CLIPImageProcessorFast")), + ("layoutlmv2", ("LayoutLMv2ImageProcessor",)), + ("layoutlmv3", ("LayoutLMv3ImageProcessor",)), + ("levit", ("LevitImageProcessor",)), + ("llama4", ("Llama4ImageProcessor", "Llama4ImageProcessorFast")), + ("llava", ("LlavaImageProcessor", "LlavaImageProcessorFast")), + ("llava_next", ("LlavaNextImageProcessor", "LlavaNextImageProcessorFast")), + ("llava_next_video", ("LlavaNextVideoImageProcessor",)), + ("llava_onevision", ("LlavaOnevisionImageProcessor", "LlavaOnevisionImageProcessorFast")), + ("mask2former", ("Mask2FormerImageProcessor",)), + ("maskformer", ("MaskFormerImageProcessor",)), + ("mgp-str", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("mistral3", ("PixtralImageProcessor", "PixtralImageProcessorFast")), + ("mllama", ("MllamaImageProcessor",)), + ("mobilenet_v1", ("MobileNetV1ImageProcessor",)), + ("mobilenet_v2", ("MobileNetV2ImageProcessor",)), + ("mobilevit", ("MobileViTImageProcessor",)), + ("mobilevitv2", ("MobileViTImageProcessor",)), + ("nat", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("nougat", ("NougatImageProcessor",)), + ("oneformer", ("OneFormerImageProcessor",)), + ("owlv2", ("Owlv2ImageProcessor",)), + ("owlvit", ("OwlViTImageProcessor",)), + ("paligemma", ("SiglipImageProcessor", "SiglipImageProcessorFast")), + ("perceiver", ("PerceiverImageProcessor",)), + ("phi4_multimodal", "Phi4MultimodalImageProcessorFast"), + ("pix2struct", ("Pix2StructImageProcessor",)), + ("pixtral", ("PixtralImageProcessor", "PixtralImageProcessorFast")), + ("poolformer", ("PoolFormerImageProcessor",)), + ("prompt_depth_anything", ("PromptDepthAnythingImageProcessor",)), + ("pvt", ("PvtImageProcessor",)), + ("pvt_v2", ("PvtImageProcessor",)), + ("qwen2_5_vl", ("Qwen2VLImageProcessor", "Qwen2VLImageProcessorFast")), + ("qwen2_vl", ("Qwen2VLImageProcessor", "Qwen2VLImageProcessorFast")), + ("regnet", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), + ("resnet", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), + ("rt_detr", ("RTDetrImageProcessor", "RTDetrImageProcessorFast")), + ("sam", ("SamImageProcessor",)), + ("segformer", ("SegformerImageProcessor",)), + ("seggpt", ("SegGptImageProcessor",)), + ("shieldgemma2", ("Gemma3ImageProcessor", "Gemma3ImageProcessorFast")), + ("siglip", ("SiglipImageProcessor", "SiglipImageProcessorFast")), + ("siglip2", ("Siglip2ImageProcessor", "Siglip2ImageProcessorFast")), + ("superglue", ("SuperGlueImageProcessor",)), + ("swiftformer", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("swin", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("swin2sr", ("Swin2SRImageProcessor",)), + ("swinv2", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("table-transformer", ("DetrImageProcessor",)), + ("timesformer", ("VideoMAEImageProcessor",)), + ("timm_wrapper", ("TimmWrapperImageProcessor",)), + ("tvlt", ("TvltImageProcessor",)), + ("tvp", ("TvpImageProcessor",)), + ("udop", ("LayoutLMv3ImageProcessor",)), + ("upernet", ("SegformerImageProcessor",)), + ("van", ("ConvNextImageProcessor", "ConvNextImageProcessorFast")), + ("videomae", ("VideoMAEImageProcessor",)), + ("vilt", ("ViltImageProcessor",)), + ("vipllava", ("CLIPImageProcessor", "CLIPImageProcessorFast")), + ("vit", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("vit_hybrid", ("ViTHybridImageProcessor",)), + ("vit_mae", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("vit_msn", ("ViTImageProcessor", "ViTImageProcessorFast")), + ("vitmatte", ("VitMatteImageProcessor",)), + ("xclip", ("CLIPImageProcessor", "CLIPImageProcessorFast")), + ("yolos", ("YolosImageProcessor",)), + ("zoedepth", ("ZoeDepthImageProcessor",)), + ] + ) + +for model_type, image_processors in IMAGE_PROCESSOR_MAPPING_NAMES.items(): + slow_image_processor_class, *fast_image_processor_class = image_processors + if not is_vision_available(): + slow_image_processor_class = None + + # If the fast image processor is not defined, or torchvision is not available, we set it to None + if not fast_image_processor_class or fast_image_processor_class[0] is None or not is_torchvision_available(): + fast_image_processor_class = None + else: + fast_image_processor_class = fast_image_processor_class[0] + + IMAGE_PROCESSOR_MAPPING_NAMES[model_type] = (slow_image_processor_class, fast_image_processor_class) + +IMAGE_PROCESSOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES) + + +def get_image_processor_class_from_name(class_name: str): + if class_name == "BaseImageProcessorFast": + return BaseImageProcessorFast + + for module_name, extractors in IMAGE_PROCESSOR_MAPPING_NAMES.items(): + if class_name in extractors: + module_name = model_type_to_module_name(module_name) + + module = importlib.import_module(f".{module_name}", "transformers.models") + try: + return getattr(module, class_name) + except AttributeError: + continue + + for _, extractors in IMAGE_PROCESSOR_MAPPING._extra_content.items(): + for extractor in extractors: + if getattr(extractor, "__name__", None) == class_name: + return extractor + + # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main + # init and we return the proper dummy to get an appropriate error message. + main_module = importlib.import_module("transformers") + if hasattr(main_module, class_name): + return getattr(main_module, class_name) + + return None + + +def get_image_processor_config( + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + resume_download: Optional[bool] = None, + proxies: Optional[Dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + local_files_only: bool = False, + **kwargs, +): + """ + Loads the image processor configuration from a pretrained model image processor configuration. + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a configuration file saved using the + [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. + + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the image processor configuration from local files. + + + + Passing `token=True` is required when you want to use a private model. + + + + Returns: + `Dict`: The configuration of the image processor. + + Examples: + + ```python + # Download configuration from huggingface.co and cache. + image_processor_config = get_image_processor_config("google-bert/bert-base-uncased") + # This model does not have a image processor config so the result will be an empty dict. + image_processor_config = get_image_processor_config("FacebookAI/xlm-roberta-base") + + # Save a pretrained image processor locally and you can reload its config + from transformers import AutoTokenizer + + image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") + image_processor.save_pretrained("image-processor-test") + image_processor_config = get_image_processor_config("image-processor-test") + ```""" + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + token = use_auth_token + + resolved_config_file = cached_file( + pretrained_model_name_or_path, + IMAGE_PROCESSOR_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + _raise_exceptions_for_gated_repo=False, + _raise_exceptions_for_missing_entries=False, + _raise_exceptions_for_connection_errors=False, + ) + if resolved_config_file is None: + logger.info( + "Could not locate the image processor configuration file, will try to use the model config instead." + ) + return {} + + with open(resolved_config_file, encoding="utf-8") as reader: + return json.load(reader) + + +def _warning_fast_image_processor_available(fast_class): + logger.warning( + f"Fast image processor class {fast_class} is available for this model. " + "Using slow image processor class. To use the fast image processor class set `use_fast=True`." + ) + + +class AutoImageProcessor: + r""" + This is a generic image processor class that will be instantiated as one of the image processor classes of the + library when created with the [`AutoImageProcessor.from_pretrained`] class method. + + This class cannot be instantiated directly using `__init__()` (throws an error). + """ + + def __init__(self): + raise EnvironmentError( + "AutoImageProcessor is designed to be instantiated " + "using the `AutoImageProcessor.from_pretrained(pretrained_model_name_or_path)` method." + ) + + @classmethod + @replace_list_option_in_docstrings(IMAGE_PROCESSOR_MAPPING_NAMES) + def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): + r""" + Instantiate one of the image processor classes of the library from a pretrained model vocabulary. + + The image processor class to instantiate is selected based on the `model_type` property of the config object + (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's + missing, by falling back to using pattern matching on `pretrained_model_name_or_path`: + + List options + + Params: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained image_processor hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a image processor file saved using the + [`~image_processing_utils.ImageProcessingMixin.save_pretrained`] method, e.g., + `./my_model_directory/`. + - a path or url to a saved image processor JSON *file*, e.g., + `./my_model_directory/preprocessor_config.json`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model image processor should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the image processor files and override the cached versions if + they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + use_fast (`bool`, *optional*, defaults to `False`): + Use a fast torchvision-base image processor if it is supported for a given model. + If a fast image processor is not available for a given model, a normal numpy-based image processor + is returned instead. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + If `False`, then this function returns just the final image processor object. If `True`, then this + functions returns a `Tuple(image_processor, unused_kwargs)` where *unused_kwargs* is a dictionary + consisting of the key/value pairs whose keys are not image processor attributes: i.e., the part of + `kwargs` which has not been used to update `image_processor` and is otherwise ignored. + trust_remote_code (`bool`, *optional*, defaults to `False`): + Whether or not to allow for custom models defined on the Hub in their own modeling files. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + image_processor_filename (`str`, *optional*, defaults to `"config.json"`): + The name of the file in the model directory to use for the image processor config. + kwargs (`Dict[str, Any]`, *optional*): + The values in kwargs of any keys which are image processor attributes will be used to override the + loaded values. Behavior concerning key/value pairs whose keys are *not* image processor attributes is + controlled by the `return_unused_kwargs` keyword parameter. + + + + Passing `token=True` is required when you want to use a private model. + + + + Examples: + + ```python + >>> from transformers import AutoImageProcessor + + >>> # Download image processor from huggingface.co and cache. + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") + + >>> # If image processor files are in a directory (e.g. image processor was saved using *save_pretrained('./test/saved_model/')*) + >>> # image_processor = AutoImageProcessor.from_pretrained("./test/saved_model/") + ```""" + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if kwargs.get("token", None) is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + kwargs["token"] = use_auth_token + + config = kwargs.pop("config", None) + # TODO: @yoni, change in v4.48 (use_fast set to True by default) + use_fast = kwargs.pop("use_fast", None) + trust_remote_code = kwargs.pop("trust_remote_code", None) + kwargs["_from_auto"] = True + + # Resolve the image processor config filename + if "image_processor_filename" in kwargs: + image_processor_filename = kwargs.pop("image_processor_filename") + elif is_timm_local_checkpoint(pretrained_model_name_or_path): + image_processor_filename = CONFIG_NAME + else: + image_processor_filename = IMAGE_PROCESSOR_NAME + + # Load the image processor config + try: + # Main path for all transformers models and local TimmWrapper checkpoints + config_dict, _ = ImageProcessingMixin.get_image_processor_dict( + pretrained_model_name_or_path, image_processor_filename=image_processor_filename, **kwargs + ) + except Exception as initial_exception: + # Fallback path for Hub TimmWrapper checkpoints. Timm models' image processing is saved in `config.json` + # instead of `preprocessor_config.json`. Because this is an Auto class and we don't have any information + # except the model name, the only way to check if a remote checkpoint is a timm model is to try to + # load `config.json` and if it fails with some error, we raise the initial exception. + try: + config_dict, _ = ImageProcessingMixin.get_image_processor_dict( + pretrained_model_name_or_path, image_processor_filename=CONFIG_NAME, **kwargs + ) + except Exception: + raise initial_exception + + # In case we have a config_dict, but it's not a timm config dict, we raise the initial exception, + # because only timm models have image processing in `config.json`. + if not is_timm_config_dict(config_dict): + raise initial_exception + + image_processor_type = config_dict.get("image_processor_type", None) + image_processor_auto_map = None + if "AutoImageProcessor" in config_dict.get("auto_map", {}): + image_processor_auto_map = config_dict["auto_map"]["AutoImageProcessor"] + + # If we still don't have the image processor class, check if we're loading from a previous feature extractor config + # and if so, infer the image processor class from there. + if image_processor_type is None and image_processor_auto_map is None: + feature_extractor_class = config_dict.pop("feature_extractor_type", None) + if feature_extractor_class is not None: + image_processor_type = feature_extractor_class.replace("FeatureExtractor", "ImageProcessor") + if "AutoFeatureExtractor" in config_dict.get("auto_map", {}): + feature_extractor_auto_map = config_dict["auto_map"]["AutoFeatureExtractor"] + image_processor_auto_map = feature_extractor_auto_map.replace("FeatureExtractor", "ImageProcessor") + + # If we don't find the image processor class in the image processor config, let's try the model config. + if image_processor_type is None and image_processor_auto_map is None: + if not isinstance(config, PretrainedConfig): + config = AutoConfig.from_pretrained( + pretrained_model_name_or_path, + trust_remote_code=trust_remote_code, + **kwargs, + ) + # It could be in `config.image_processor_type`` + image_processor_type = getattr(config, "image_processor_type", None) + if hasattr(config, "auto_map") and "AutoImageProcessor" in config.auto_map: + image_processor_auto_map = config.auto_map["AutoImageProcessor"] + + image_processor_class = None + # TODO: @yoni, change logic in v4.52 (when use_fast set to True by default) + if image_processor_type is not None: + # if use_fast is not set and the processor was saved with a fast processor, we use it, otherwise we use the slow processor. + if use_fast is None: + use_fast = image_processor_type.endswith("Fast") + if not use_fast: + logger.warning_once( + "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. " + "`use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. " + "This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`." + ) + # Update class name to reflect the use_fast option. If class is not found, we fall back to the slow version. + if use_fast and not is_torchvision_available(): + logger.warning_once( + "Using `use_fast=True` but `torchvision` is not available. Falling back to the slow image processor." + ) + use_fast = False + if use_fast: + if not image_processor_type.endswith("Fast"): + image_processor_type += "Fast" + for _, image_processors in IMAGE_PROCESSOR_MAPPING_NAMES.items(): + if image_processor_type in image_processors: + break + else: + image_processor_type = image_processor_type[:-4] + use_fast = False + logger.warning_once( + "`use_fast` is set to `True` but the image processor class does not have a fast version. " + " Falling back to the slow version." + ) + image_processor_class = get_image_processor_class_from_name(image_processor_type) + else: + image_processor_type = ( + image_processor_type[:-4] if image_processor_type.endswith("Fast") else image_processor_type + ) + image_processor_class = get_image_processor_class_from_name(image_processor_type) + + has_remote_code = image_processor_auto_map is not None + has_local_code = image_processor_class is not None or type(config) in IMAGE_PROCESSOR_MAPPING + trust_remote_code = resolve_trust_remote_code( + trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code + ) + + if image_processor_auto_map is not None and not isinstance(image_processor_auto_map, tuple): + # In some configs, only the slow image processor class is stored + image_processor_auto_map = (image_processor_auto_map, None) + + if has_remote_code and trust_remote_code: + if not use_fast and image_processor_auto_map[1] is not None: + _warning_fast_image_processor_available(image_processor_auto_map[1]) + + if use_fast and image_processor_auto_map[1] is not None: + class_ref = image_processor_auto_map[1] + else: + class_ref = image_processor_auto_map[0] + image_processor_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs) + _ = kwargs.pop("code_revision", None) + if os.path.isdir(pretrained_model_name_or_path): + image_processor_class.register_for_auto_class() + return image_processor_class.from_dict(config_dict, **kwargs) + elif image_processor_class is not None: + return image_processor_class.from_dict(config_dict, **kwargs) + # Last try: we use the IMAGE_PROCESSOR_MAPPING. + elif type(config) in IMAGE_PROCESSOR_MAPPING: + image_processor_tuple = IMAGE_PROCESSOR_MAPPING[type(config)] + + image_processor_class_py, image_processor_class_fast = image_processor_tuple + + if not use_fast and image_processor_class_fast is not None: + _warning_fast_image_processor_available(image_processor_class_fast) + + if image_processor_class_fast and (use_fast or image_processor_class_py is None): + return image_processor_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + else: + if image_processor_class_py is not None: + return image_processor_class_py.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + else: + raise ValueError( + "This image processor cannot be instantiated. Please make sure you have `Pillow` installed." + ) + + raise ValueError( + f"Unrecognized image processor in {pretrained_model_name_or_path}. Should have a " + f"`image_processor_type` key in its {IMAGE_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following " + f"`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in IMAGE_PROCESSOR_MAPPING_NAMES.keys())}" + ) + + @staticmethod + def register( + config_class, + image_processor_class=None, + slow_image_processor_class=None, + fast_image_processor_class=None, + exist_ok=False, + ): + """ + Register a new image processor for this class. + + Args: + config_class ([`PretrainedConfig`]): + The configuration corresponding to the model to register. + image_processor_class ([`ImageProcessingMixin`]): The image processor to register. + """ + if image_processor_class is not None: + if slow_image_processor_class is not None: + raise ValueError("Cannot specify both image_processor_class and slow_image_processor_class") + warnings.warn( + "The image_processor_class argument is deprecated and will be removed in v4.42. Please use `slow_image_processor_class`, or `fast_image_processor_class` instead", + FutureWarning, + ) + slow_image_processor_class = image_processor_class + + if slow_image_processor_class is None and fast_image_processor_class is None: + raise ValueError("You need to specify either slow_image_processor_class or fast_image_processor_class") + if slow_image_processor_class is not None and issubclass(slow_image_processor_class, BaseImageProcessorFast): + raise ValueError("You passed a fast image processor in as the `slow_image_processor_class`.") + if fast_image_processor_class is not None and not issubclass( + fast_image_processor_class, BaseImageProcessorFast + ): + raise ValueError("The `fast_image_processor_class` should inherit from `BaseImageProcessorFast`.") + + if ( + slow_image_processor_class is not None + and fast_image_processor_class is not None + and issubclass(fast_image_processor_class, BaseImageProcessorFast) + and fast_image_processor_class.slow_image_processor_class != slow_image_processor_class + ): + raise ValueError( + "The fast processor class you are passing has a `slow_image_processor_class` attribute that is not " + "consistent with the slow processor class you passed (fast tokenizer has " + f"{fast_image_processor_class.slow_image_processor_class} and you passed {slow_image_processor_class}. Fix one of those " + "so they match!" + ) + + # Avoid resetting a set slow/fast image processor if we are passing just the other ones. + if config_class in IMAGE_PROCESSOR_MAPPING._extra_content: + existing_slow, existing_fast = IMAGE_PROCESSOR_MAPPING[config_class] + if slow_image_processor_class is None: + slow_image_processor_class = existing_slow + if fast_image_processor_class is None: + fast_image_processor_class = existing_fast + + IMAGE_PROCESSOR_MAPPING.register( + config_class, (slow_image_processor_class, fast_image_processor_class), exist_ok=exist_ok + ) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_base.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_base.py new file mode 100644 index 0000000000000000000000000000000000000000..5398abe028222b14a372e4e8d3a05a2e1dbf9883 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_base.py @@ -0,0 +1,558 @@ +# Copyright 2020 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import copy +import json +import os +import warnings +from io import BytesIO +from typing import Any, Optional, TypeVar, Union + +import numpy as np +import requests + +from .dynamic_module_utils import custom_object_save +from .feature_extraction_utils import BatchFeature as BaseBatchFeature +from .utils import ( + IMAGE_PROCESSOR_NAME, + PushToHubMixin, + add_model_info_to_auto_map, + add_model_info_to_custom_pipelines, + cached_file, + copy_func, + download_url, + is_offline_mode, + is_remote_url, + is_vision_available, + logging, +) + + +if is_vision_available(): + from PIL import Image + + +ImageProcessorType = TypeVar("ImageProcessorType", bound="ImageProcessingMixin") + + +logger = logging.get_logger(__name__) + + +# TODO: Move BatchFeature to be imported by both image_processing_utils and image_processing_utils +# We override the class string here, but logic is the same. +class BatchFeature(BaseBatchFeature): + r""" + Holds the output of the image processor specific `__call__` methods. + + This class is derived from a python dictionary and can be used as a dictionary. + + Args: + data (`dict`): + Dictionary of lists/arrays/tensors returned by the __call__ method ('pixel_values', etc.). + tensor_type (`Union[None, str, TensorType]`, *optional*): + You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at + initialization. + """ + + +# TODO: (Amy) - factor out the common parts of this and the feature extractor +class ImageProcessingMixin(PushToHubMixin): + """ + This is an image processor mixin used to provide saving/loading functionality for sequential and image feature + extractors. + """ + + _auto_class = None + + def __init__(self, **kwargs): + """Set elements of `kwargs` as attributes.""" + # This key was saved while we still used `XXXFeatureExtractor` for image processing. Now we use + # `XXXImageProcessor`, this attribute and its value are misleading. + kwargs.pop("feature_extractor_type", None) + # Pop "processor_class" as it should be saved as private attribute + self._processor_class = kwargs.pop("processor_class", None) + # Additional attributes without default values + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error(f"Can't set {key} with value {value} for {self}") + raise err + + def _set_processor_class(self, processor_class: str): + """Sets processor class as an attribute.""" + self._processor_class = processor_class + + @classmethod + def from_pretrained( + cls: type[ImageProcessorType], + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ) -> ImageProcessorType: + r""" + Instantiate a type of [`~image_processing_utils.ImageProcessingMixin`] from an image processor. + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained image_processor hosted inside a model repo on + huggingface.co. + - a path to a *directory* containing a image processor file saved using the + [`~image_processing_utils.ImageProcessingMixin.save_pretrained`] method, e.g., + `./my_model_directory/`. + - a path or url to a saved image processor JSON *file*, e.g., + `./my_model_directory/preprocessor_config.json`. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model image processor should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the image processor files and override the cached versions if + they exist. + resume_download: + Deprecated and ignored. All downloads are now resumed by default when possible. + Will be removed in v5 of Transformers. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use + the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + + + + + To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. + + + + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + If `False`, then this function returns just the final image processor object. If `True`, then this + functions returns a `Tuple(image_processor, unused_kwargs)` where *unused_kwargs* is a dictionary + consisting of the key/value pairs whose keys are not image processor attributes: i.e., the part of + `kwargs` which has not been used to update `image_processor` and is otherwise ignored. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can + specify the folder name here. + kwargs (`Dict[str, Any]`, *optional*): + The values in kwargs of any keys which are image processor attributes will be used to override the + loaded values. Behavior concerning key/value pairs whose keys are *not* image processor attributes is + controlled by the `return_unused_kwargs` keyword parameter. + + Returns: + A image processor of type [`~image_processing_utils.ImageProcessingMixin`]. + + Examples: + + ```python + # We can't instantiate directly the base class *ImageProcessingMixin* so let's show the examples on a + # derived class: *CLIPImageProcessor* + image_processor = CLIPImageProcessor.from_pretrained( + "openai/clip-vit-base-patch32" + ) # Download image_processing_config from huggingface.co and cache. + image_processor = CLIPImageProcessor.from_pretrained( + "./test/saved_model/" + ) # E.g. image processor (or model) was saved using *save_pretrained('./test/saved_model/')* + image_processor = CLIPImageProcessor.from_pretrained("./test/saved_model/preprocessor_config.json") + image_processor = CLIPImageProcessor.from_pretrained( + "openai/clip-vit-base-patch32", do_normalize=False, foo=False + ) + assert image_processor.do_normalize is False + image_processor, unused_kwargs = CLIPImageProcessor.from_pretrained( + "openai/clip-vit-base-patch32", do_normalize=False, foo=False, return_unused_kwargs=True + ) + assert image_processor.do_normalize is False + assert unused_kwargs == {"foo": False} + ```""" + kwargs["cache_dir"] = cache_dir + kwargs["force_download"] = force_download + kwargs["local_files_only"] = local_files_only + kwargs["revision"] = revision + + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + kwargs["token"] = token + + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + + return cls.from_dict(image_processor_dict, **kwargs) + + def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save an image processor object to the directory `save_directory`, so that it can be re-loaded using the + [`~image_processing_utils.ImageProcessingMixin.from_pretrained`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the image processor JSON file will be saved (will be created if it does not exist). + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + use_auth_token = kwargs.pop("use_auth_token", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if kwargs.get("token", None) is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + kwargs["token"] = use_auth_token + + if os.path.isfile(save_directory): + raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file") + + os.makedirs(save_directory, exist_ok=True) + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = self._create_repo(repo_id, **kwargs) + files_timestamps = self._get_files_timestamps(save_directory) + + # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be + # loaded from the Hub. + if self._auto_class is not None: + custom_object_save(self, save_directory, config=self) + + # If we save using the predefined names, we can load using `from_pretrained` + output_image_processor_file = os.path.join(save_directory, IMAGE_PROCESSOR_NAME) + + self.to_json_file(output_image_processor_file) + logger.info(f"Image processor saved in {output_image_processor_file}") + + if push_to_hub: + self._upload_modified_files( + save_directory, + repo_id, + files_timestamps, + commit_message=commit_message, + token=kwargs.get("token"), + ) + + return [output_image_processor_file] + + @classmethod + def get_image_processor_dict( + cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs + ) -> tuple[dict[str, Any], dict[str, Any]]: + """ + From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating a + image processor of type [`~image_processor_utils.ImageProcessingMixin`] using `from_dict`. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`): + The identifier of the pre-trained checkpoint from which we want the dictionary of parameters. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can + specify the folder name here. + image_processor_filename (`str`, *optional*, defaults to `"config.json"`): + The name of the file in the model directory to use for the image processor config. + + Returns: + `Tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the image processor object. + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", None) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + subfolder = kwargs.pop("subfolder", "") + image_processor_filename = kwargs.pop("image_processor_filename", IMAGE_PROCESSOR_NAME) + + from_pipeline = kwargs.pop("_from_pipeline", None) + from_auto_class = kwargs.pop("_from_auto", False) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.", + FutureWarning, + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + user_agent = {"file_type": "image processor", "from_auto_class": from_auto_class} + if from_pipeline is not None: + user_agent["using_pipeline"] = from_pipeline + + if is_offline_mode() and not local_files_only: + logger.info("Offline mode: forcing local_files_only=True") + local_files_only = True + + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + is_local = os.path.isdir(pretrained_model_name_or_path) + if os.path.isdir(pretrained_model_name_or_path): + image_processor_file = os.path.join(pretrained_model_name_or_path, image_processor_filename) + if os.path.isfile(pretrained_model_name_or_path): + resolved_image_processor_file = pretrained_model_name_or_path + is_local = True + elif is_remote_url(pretrained_model_name_or_path): + image_processor_file = pretrained_model_name_or_path + resolved_image_processor_file = download_url(pretrained_model_name_or_path) + else: + image_processor_file = image_processor_filename + try: + # Load from local folder or from cache or download from model Hub and cache + resolved_image_processor_file = cached_file( + pretrained_model_name_or_path, + image_processor_file, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + revision=revision, + subfolder=subfolder, + ) + except OSError: + # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to + # the original exception. + raise + except Exception: + # For any other exception, we throw a generic error. + raise OSError( + f"Can't load image processor for '{pretrained_model_name_or_path}'. If you were trying to load" + " it from 'https://huggingface.co/models', make sure you don't have a local directory with the" + f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a" + f" directory containing a {image_processor_filename} file" + ) + + try: + # Load image_processor dict + with open(resolved_image_processor_file, encoding="utf-8") as reader: + text = reader.read() + image_processor_dict = json.loads(text) + + except json.JSONDecodeError: + raise OSError( + f"It looks like the config file at '{resolved_image_processor_file}' is not a valid JSON file." + ) + + if is_local: + logger.info(f"loading configuration file {resolved_image_processor_file}") + else: + logger.info( + f"loading configuration file {image_processor_file} from cache at {resolved_image_processor_file}" + ) + if "auto_map" in image_processor_dict: + image_processor_dict["auto_map"] = add_model_info_to_auto_map( + image_processor_dict["auto_map"], pretrained_model_name_or_path + ) + if "custom_pipelines" in image_processor_dict: + image_processor_dict["custom_pipelines"] = add_model_info_to_custom_pipelines( + image_processor_dict["custom_pipelines"], pretrained_model_name_or_path + ) + + return image_processor_dict, kwargs + + @classmethod + def from_dict(cls, image_processor_dict: dict[str, Any], **kwargs): + """ + Instantiates a type of [`~image_processing_utils.ImageProcessingMixin`] from a Python dictionary of parameters. + + Args: + image_processor_dict (`Dict[str, Any]`): + Dictionary that will be used to instantiate the image processor object. Such a dictionary can be + retrieved from a pretrained checkpoint by leveraging the + [`~image_processing_utils.ImageProcessingMixin.to_dict`] method. + kwargs (`Dict[str, Any]`): + Additional parameters from which to initialize the image processor object. + + Returns: + [`~image_processing_utils.ImageProcessingMixin`]: The image processor object instantiated from those + parameters. + """ + image_processor_dict = image_processor_dict.copy() + return_unused_kwargs = kwargs.pop("return_unused_kwargs", False) + + # The `size` parameter is a dict and was previously an int or tuple in feature extractors. + # We set `size` here directly to the `image_processor_dict` so that it is converted to the appropriate + # dict within the image processor and isn't overwritten if `size` is passed in as a kwarg. + if "size" in kwargs and "size" in image_processor_dict: + image_processor_dict["size"] = kwargs.pop("size") + if "crop_size" in kwargs and "crop_size" in image_processor_dict: + image_processor_dict["crop_size"] = kwargs.pop("crop_size") + + image_processor = cls(**image_processor_dict) + + # Update image_processor with kwargs if needed + to_remove = [] + for key, value in kwargs.items(): + if hasattr(image_processor, key): + setattr(image_processor, key, value) + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + + logger.info(f"Image processor {image_processor}") + if return_unused_kwargs: + return image_processor, kwargs + else: + return image_processor + + def to_dict(self) -> dict[str, Any]: + """ + Serializes this instance to a Python dictionary. + + Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this image processor instance. + """ + output = copy.deepcopy(self.__dict__) + output["image_processor_type"] = self.__class__.__name__ + + return output + + @classmethod + def from_json_file(cls, json_file: Union[str, os.PathLike]): + """ + Instantiates a image processor of type [`~image_processing_utils.ImageProcessingMixin`] from the path to a JSON + file of parameters. + + Args: + json_file (`str` or `os.PathLike`): + Path to the JSON file containing the parameters. + + Returns: + A image processor of type [`~image_processing_utils.ImageProcessingMixin`]: The image_processor object + instantiated from that JSON file. + """ + with open(json_file, encoding="utf-8") as reader: + text = reader.read() + image_processor_dict = json.loads(text) + return cls(**image_processor_dict) + + def to_json_string(self) -> str: + """ + Serializes this instance to a JSON string. + + Returns: + `str`: String containing all the attributes that make up this feature_extractor instance in JSON format. + """ + dictionary = self.to_dict() + + for key, value in dictionary.items(): + if isinstance(value, np.ndarray): + dictionary[key] = value.tolist() + + # make sure private name "_processor_class" is correctly + # saved as "processor_class" + _processor_class = dictionary.pop("_processor_class", None) + if _processor_class is not None: + dictionary["processor_class"] = _processor_class + + return json.dumps(dictionary, indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this image_processor instance's parameters will be saved. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + writer.write(self.to_json_string()) + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + @classmethod + def register_for_auto_class(cls, auto_class="AutoImageProcessor"): + """ + Register this class with a given auto class. This should only be used for custom image processors as the ones + in the library are already mapped with `AutoImageProcessor `. + + + + This API is experimental and may have some slight breaking changes in the next releases. + + + + Args: + auto_class (`str` or `type`, *optional*, defaults to `"AutoImageProcessor "`): + The auto class to register this new image processor with. + """ + if not isinstance(auto_class, str): + auto_class = auto_class.__name__ + + import transformers.models.auto as auto_module + + if not hasattr(auto_module, auto_class): + raise ValueError(f"{auto_class} is not a valid auto class.") + + cls._auto_class = auto_class + + def fetch_images(self, image_url_or_urls: Union[str, list[str]]): + """ + Convert a single or a list of urls into the corresponding `PIL.Image` objects. + + If a single url is passed, the return value will be a single object. If a list is passed a list of objects is + returned. + """ + headers = { + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0" + " Safari/537.36" + ) + } + if isinstance(image_url_or_urls, list): + return [self.fetch_images(x) for x in image_url_or_urls] + elif isinstance(image_url_or_urls, str): + response = requests.get(image_url_or_urls, stream=True, headers=headers) + response.raise_for_status() + return Image.open(BytesIO(response.content)) + else: + raise TypeError(f"only a single or a list of entries is supported but got type={type(image_url_or_urls)}") + + +ImageProcessingMixin.push_to_hub = copy_func(ImageProcessingMixin.push_to_hub) +if ImageProcessingMixin.push_to_hub.__doc__ is not None: + ImageProcessingMixin.push_to_hub.__doc__ = ImageProcessingMixin.push_to_hub.__doc__.format( + object="image processor", object_class="AutoImageProcessor", object_files="image processor file" + ) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ec0f817728d327fe95b140f3f30f33bac7042ad9 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_utils.py @@ -0,0 +1,308 @@ +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from collections.abc import Iterable +from typing import Optional, Union + +import numpy as np + +from .image_processing_base import BatchFeature, ImageProcessingMixin +from .image_transforms import center_crop, normalize, rescale +from .image_utils import ChannelDimension, get_image_size +from .utils import logging + + +logger = logging.get_logger(__name__) + + +INIT_SERVICE_KWARGS = [ + "processor_class", + "image_processor_type", +] + + +class BaseImageProcessor(ImageProcessingMixin): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def __call__(self, images, **kwargs) -> BatchFeature: + """Preprocess an image or a batch of images.""" + return self.preprocess(images, **kwargs) + + def preprocess(self, images, **kwargs) -> BatchFeature: + raise NotImplementedError("Each image processor must implement its own preprocess method") + + def rescale( + self, + image: np.ndarray, + scale: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, input_data_format=input_data_format, **kwargs) + + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize( + image, mean=mean, std=std, data_format=data_format, input_data_format=input_data_format, **kwargs + ) + + def center_crop( + self, + image: np.ndarray, + size: dict[str, int], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Center crop an image to `(size["height"], size["width"])`. If the input size is smaller than `crop_size` along + any edge, the image is padded with 0's and then center cropped. + + Args: + image (`np.ndarray`): + Image to center crop. + size (`Dict[str, int]`): + Size of the output image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + """ + size = get_size_dict(size) + if "height" not in size or "width" not in size: + raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}") + return center_crop( + image, + size=(size["height"], size["width"]), + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) + + def to_dict(self): + encoder_dict = super().to_dict() + encoder_dict.pop("_valid_processor_keys", None) + return encoder_dict + + +VALID_SIZE_DICT_KEYS = ( + {"height", "width"}, + {"shortest_edge"}, + {"shortest_edge", "longest_edge"}, + {"longest_edge"}, + {"max_height", "max_width"}, +) + + +def is_valid_size_dict(size_dict): + if not isinstance(size_dict, dict): + return False + + size_dict_keys = set(size_dict.keys()) + for allowed_keys in VALID_SIZE_DICT_KEYS: + if size_dict_keys == allowed_keys: + return True + return False + + +def convert_to_size_dict( + size, max_size: Optional[int] = None, default_to_square: bool = True, height_width_order: bool = True +): + # By default, if size is an int we assume it represents a tuple of (size, size). + if isinstance(size, int) and default_to_square: + if max_size is not None: + raise ValueError("Cannot specify both size as an int, with default_to_square=True and max_size") + return {"height": size, "width": size} + # In other configs, if size is an int and default_to_square is False, size represents the length of + # the shortest edge after resizing. + elif isinstance(size, int) and not default_to_square: + size_dict = {"shortest_edge": size} + if max_size is not None: + size_dict["longest_edge"] = max_size + return size_dict + # Otherwise, if size is a tuple it's either (height, width) or (width, height) + elif isinstance(size, (tuple, list)) and height_width_order: + return {"height": size[0], "width": size[1]} + elif isinstance(size, (tuple, list)) and not height_width_order: + return {"height": size[1], "width": size[0]} + elif size is None and max_size is not None: + if default_to_square: + raise ValueError("Cannot specify both default_to_square=True and max_size") + return {"longest_edge": max_size} + + raise ValueError(f"Could not convert size input to size dict: {size}") + + +def get_size_dict( + size: Union[int, Iterable[int], dict[str, int]] = None, + max_size: Optional[int] = None, + height_width_order: bool = True, + default_to_square: bool = True, + param_name="size", +) -> dict: + """ + Converts the old size parameter in the config into the new dict expected in the config. This is to ensure backwards + compatibility with the old image processor configs and removes ambiguity over whether the tuple is in (height, + width) or (width, height) format. + + - If `size` is tuple, it is converted to `{"height": size[0], "width": size[1]}` or `{"height": size[1], "width": + size[0]}` if `height_width_order` is `False`. + - If `size` is an int, and `default_to_square` is `True`, it is converted to `{"height": size, "width": size}`. + - If `size` is an int and `default_to_square` is False, it is converted to `{"shortest_edge": size}`. If `max_size` + is set, it is added to the dict as `{"longest_edge": max_size}`. + + Args: + size (`Union[int, Iterable[int], Dict[str, int]]`, *optional*): + The `size` parameter to be cast into a size dictionary. + max_size (`Optional[int]`, *optional*): + The `max_size` parameter to be cast into a size dictionary. + height_width_order (`bool`, *optional*, defaults to `True`): + If `size` is a tuple, whether it's in (height, width) or (width, height) order. + default_to_square (`bool`, *optional*, defaults to `True`): + If `size` is an int, whether to default to a square image or not. + """ + if not isinstance(size, dict): + size_dict = convert_to_size_dict(size, max_size, default_to_square, height_width_order) + logger.info( + f"{param_name} should be a dictionary on of the following set of keys: {VALID_SIZE_DICT_KEYS}, got {size}." + f" Converted to {size_dict}.", + ) + else: + size_dict = size + + if not is_valid_size_dict(size_dict): + raise ValueError( + f"{param_name} must have one of the following set of keys: {VALID_SIZE_DICT_KEYS}, got {size_dict.keys()}" + ) + return size_dict + + +def select_best_resolution(original_size: tuple, possible_resolutions: list) -> tuple: + """ + Selects the best resolution from a list of possible resolutions based on the original size. + + This is done by calculating the effective and wasted resolution for each possible resolution. + + The best fit resolution is the one that maximizes the effective resolution and minimizes the wasted resolution. + + Args: + original_size (tuple): + The original size of the image in the format (height, width). + possible_resolutions (list): + A list of possible resolutions in the format [(height1, width1), (height2, width2), ...]. + + Returns: + tuple: The best fit resolution in the format (height, width). + """ + original_height, original_width = original_size + best_fit = None + max_effective_resolution = 0 + min_wasted_resolution = float("inf") + + for height, width in possible_resolutions: + scale = min(width / original_width, height / original_height) + downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale) + effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height) + wasted_resolution = (width * height) - effective_resolution + + if effective_resolution > max_effective_resolution or ( + effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution + ): + max_effective_resolution = effective_resolution + min_wasted_resolution = wasted_resolution + best_fit = (height, width) + + return best_fit + + +def get_patch_output_size(image, target_resolution, input_data_format): + """ + Given an image and a target resolution, calculate the output size of the image after cropping to the target + """ + original_height, original_width = get_image_size(image, channel_dim=input_data_format) + target_height, target_width = target_resolution + + scale_w = target_width / original_width + scale_h = target_height / original_height + + if scale_w < scale_h: + new_width = target_width + new_height = min(math.ceil(original_height * scale_w), target_height) + else: + new_height = target_height + new_width = min(math.ceil(original_width * scale_h), target_width) + + return new_height, new_width diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_utils_fast.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_utils_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..b671a1119111353f6569e4ba0f379c8e113de70b --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_processing_utils_fast.py @@ -0,0 +1,792 @@ +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Iterable +from functools import lru_cache, partial +from typing import Any, Optional, TypedDict, Union + +import numpy as np + +from .image_processing_utils import ( + BaseImageProcessor, + BatchFeature, + get_size_dict, +) +from .image_transforms import ( + convert_to_rgb, + get_resize_output_image_size, + get_size_with_aspect_ratio, + group_images_by_shape, + reorder_images, +) +from .image_utils import ( + ChannelDimension, + ImageInput, + ImageType, + SizeDict, + get_image_size, + get_image_size_for_max_height_width, + get_image_type, + infer_channel_dimension_format, + make_flat_list_of_images, + validate_kwargs, + validate_preprocess_arguments, +) +from .processing_utils import Unpack +from .utils import ( + TensorType, + add_start_docstrings, + is_torch_available, + is_torchvision_available, + is_torchvision_v2_available, + is_vision_available, + logging, +) + + +if is_vision_available(): + from .image_utils import PILImageResampling + +if is_torch_available(): + import torch + +if is_torchvision_available(): + from .image_utils import pil_torch_interpolation_mapping + + if is_torchvision_v2_available(): + from torchvision.transforms.v2 import functional as F + else: + from torchvision.transforms import functional as F + +logger = logging.get_logger(__name__) + + +@lru_cache(maxsize=10) +def validate_fast_preprocess_arguments( + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + do_pad: Optional[bool] = None, + size_divisibility: Optional[int] = None, + do_center_crop: Optional[bool] = None, + crop_size: Optional[SizeDict] = None, + do_resize: Optional[bool] = None, + size: Optional[SizeDict] = None, + resample: Optional["PILImageResampling"] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, +): + """ + Checks validity of typically used arguments in an `ImageProcessorFast` `preprocess` method. + Raises `ValueError` if arguments incompatibility is caught. + """ + validate_preprocess_arguments( + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + do_pad=do_pad, + size_divisibility=size_divisibility, + do_center_crop=do_center_crop, + crop_size=crop_size, + do_resize=do_resize, + size=size, + resample=resample, + ) + # Extra checks for ImageProcessorFast + if return_tensors is not None and return_tensors != "pt": + raise ValueError("Only returning PyTorch tensors is currently supported.") + + if data_format != ChannelDimension.FIRST: + raise ValueError("Only channel first data format is currently supported.") + + +def safe_squeeze(tensor: "torch.Tensor", axis: Optional[int] = None) -> "torch.Tensor": + """ + Squeezes a tensor, but only if the axis specified has dim 1. + """ + if axis is None: + return tensor.squeeze() + + try: + return tensor.squeeze(axis=axis) + except ValueError: + return tensor + + +def max_across_indices(values: Iterable[Any]) -> list[Any]: + """ + Return the maximum value across all indices of an iterable of values. + """ + return [max(values_i) for values_i in zip(*values)] + + +def get_max_height_width(images: list["torch.Tensor"]) -> tuple[int]: + """ + Get the maximum height and width across all images in a batch. + """ + + _, max_height, max_width = max_across_indices([img.shape for img in images]) + + return (max_height, max_width) + + +def divide_to_patches( + image: Union[np.array, "torch.Tensor"], patch_size: int +) -> list[Union[np.array, "torch.Tensor"]]: + """ + Divides an image into patches of a specified size. + + Args: + image (`Union[np.array, "torch.Tensor"]`): + The input image. + patch_size (`int`): + The size of each patch. + Returns: + list: A list of Union[np.array, "torch.Tensor"] representing the patches. + """ + patches = [] + height, width = get_image_size(image, channel_dim=ChannelDimension.FIRST) + for i in range(0, height, patch_size): + for j in range(0, width, patch_size): + patch = image[:, i : i + patch_size, j : j + patch_size] + patches.append(patch) + + return patches + + +class DefaultFastImageProcessorKwargs(TypedDict, total=False): + do_resize: Optional[bool] + size: Optional[dict[str, int]] + default_to_square: Optional[bool] + resample: Optional[Union["PILImageResampling", "F.InterpolationMode"]] + do_center_crop: Optional[bool] + crop_size: Optional[dict[str, int]] + do_rescale: Optional[bool] + rescale_factor: Optional[Union[int, float]] + do_normalize: Optional[bool] + image_mean: Optional[Union[float, list[float]]] + image_std: Optional[Union[float, list[float]]] + do_convert_rgb: Optional[bool] + return_tensors: Optional[Union[str, TensorType]] + data_format: Optional[ChannelDimension] + input_data_format: Optional[Union[str, ChannelDimension]] + device: Optional["torch.device"] + + +BASE_IMAGE_PROCESSOR_FAST_DOCSTRING = r""" + + Args: + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the + `do_resize` parameter in the `preprocess` method. + size (`dict`, *optional*, defaults to `self.size`): + Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess` + method. + default_to_square (`bool`, *optional*, defaults to `self.default_to_square`): + Whether to default to a square image when resizing, if size is an int. + resample (`PILImageResampling`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be + overridden by the `resample` parameter in the `preprocess` method. + do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): + Whether to center crop the image to the specified `crop_size`. Can be overridden by `do_center_crop` in the + `preprocess` method. + crop_size (`Dict[str, int]` *optional*, defaults to `self.crop_size`): + Size of the output image after applying `center_crop`. Can be overridden by `crop_size` in the `preprocess` + method. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the + `do_rescale` parameter in the `preprocess` method. + rescale_factor (`int` or `float`, *optional*, defaults to `self.rescale_factor`): + Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be + overridden by the `rescale_factor` parameter in the `preprocess` method. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` + method. Can be overridden by the `do_normalize` parameter in the `preprocess` method. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be + overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + Can be overridden by the `image_std` parameter in the `preprocess` method. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + return_tensors (`str` or `TensorType`, *optional*, defaults to `self.return_tensors`): + Returns stacked tensors if set to `pt, otherwise returns a list of tensors. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.data_format`): + Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors. + input_data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.input_data_format`): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + device (`torch.device`, *optional*, defaults to `self.device`): + The device to process the images on. If unset, the device is inferred from the input images.""" + +BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS = r""" + Preprocess an image or batch of images. + + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + size (`Dict[str, int]`, *optional*, defaults to `self.size`): + Describes the maximum input dimensions to the model. + resample (`PILImageResampling` or `InterpolationMode`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only + has an effect if `do_resize` is set to `True`. + do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): + Whether to center crop the image. + crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`): + Size of the output image after applying `center_crop`. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Rescale factor to rescale the image by if `do_rescale` is set to `True`. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to + `True`. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + return_tensors (`str` or `TensorType`, *optional*, defaults to `self.return_tensors`): + Returns stacked tensors if set to `pt, otherwise returns a list of tensors. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.data_format`): + Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors. + input_data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.input_data_format`): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + device (`torch.device`, *optional*, defaults to `self.device`): + The device to process the images on. If unset, the device is inferred from the input images.""" + + +@add_start_docstrings( + "Constructs a fast base image processor.", + BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, +) +class BaseImageProcessorFast(BaseImageProcessor): + resample = None + image_mean = None + image_std = None + size = None + default_to_square = True + crop_size = None + do_resize = None + do_center_crop = None + do_rescale = None + rescale_factor = 1 / 255 + do_normalize = None + do_convert_rgb = None + return_tensors = None + data_format = ChannelDimension.FIRST + input_data_format = None + device = None + model_input_names = ["pixel_values"] + valid_kwargs = DefaultFastImageProcessorKwargs + unused_kwargs = None + + def __init__( + self, + **kwargs: Unpack[DefaultFastImageProcessorKwargs], + ) -> None: + super().__init__(**kwargs) + kwargs = self.filter_out_unused_kwargs(kwargs) + size = kwargs.pop("size", self.size) + self.size = ( + get_size_dict(size=size, default_to_square=kwargs.pop("default_to_square", self.default_to_square)) + if size is not None + else None + ) + crop_size = kwargs.pop("crop_size", self.crop_size) + self.crop_size = get_size_dict(crop_size, param_name="crop_size") if crop_size is not None else None + for key in self.valid_kwargs.__annotations__.keys(): + kwarg = kwargs.pop(key, None) + if kwarg is not None: + setattr(self, key, kwarg) + else: + setattr(self, key, getattr(self, key, None)) + + def resize( + self, + image: "torch.Tensor", + size: SizeDict, + interpolation: "F.InterpolationMode" = None, + antialias: bool = True, + **kwargs, + ) -> "torch.Tensor": + """ + Resize an image to `(size["height"], size["width"])`. + + Args: + image (`torch.Tensor`): + Image to resize. + size (`SizeDict`): + Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. + resample (`InterpolationMode`, *optional*, defaults to `InterpolationMode.BILINEAR`): + `InterpolationMode` filter to use when resizing the image e.g. `InterpolationMode.BICUBIC`. + + Returns: + `torch.Tensor`: The resized image. + """ + interpolation = interpolation if interpolation is not None else F.InterpolationMode.BILINEAR + if size.shortest_edge and size.longest_edge: + # Resize the image so that the shortest edge or the longest edge is of the given size + # while maintaining the aspect ratio of the original image. + new_size = get_size_with_aspect_ratio( + image.size()[-2:], + size.shortest_edge, + size.longest_edge, + ) + elif size.shortest_edge: + new_size = get_resize_output_image_size( + image, + size=size.shortest_edge, + default_to_square=False, + input_data_format=ChannelDimension.FIRST, + ) + elif size.max_height and size.max_width: + new_size = get_image_size_for_max_height_width(image.size()[-2:], size.max_height, size.max_width) + elif size.height and size.width: + new_size = (size.height, size.width) + else: + raise ValueError( + "Size must contain 'height' and 'width' keys, or 'max_height' and 'max_width', or 'shortest_edge' key. Got" + f" {size}." + ) + return F.resize(image, new_size, interpolation=interpolation, antialias=antialias) + + def rescale( + self, + image: "torch.Tensor", + scale: float, + **kwargs, + ) -> "torch.Tensor": + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`torch.Tensor`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + + Returns: + `torch.Tensor`: The rescaled image. + """ + return image * scale + + def normalize( + self, + image: "torch.Tensor", + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + **kwargs, + ) -> "torch.Tensor": + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`torch.Tensor`): + Image to normalize. + mean (`torch.Tensor`, `float` or `Iterable[float]`): + Image mean to use for normalization. + std (`torch.Tensor`, `float` or `Iterable[float]`): + Image standard deviation to use for normalization. + + Returns: + `torch.Tensor`: The normalized image. + """ + return F.normalize(image, mean, std) + + @lru_cache(maxsize=10) + def _fuse_mean_std_and_rescale_factor( + self, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + device: Optional["torch.device"] = None, + ) -> tuple: + if do_rescale and do_normalize: + # Fused rescale and normalize + image_mean = torch.tensor(image_mean, device=device) * (1.0 / rescale_factor) + image_std = torch.tensor(image_std, device=device) * (1.0 / rescale_factor) + do_rescale = False + return image_mean, image_std, do_rescale + + def rescale_and_normalize( + self, + images: "torch.Tensor", + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Union[float, list[float]], + image_std: Union[float, list[float]], + ) -> "torch.Tensor": + """ + Rescale and normalize images. + """ + image_mean, image_std, do_rescale = self._fuse_mean_std_and_rescale_factor( + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + do_rescale=do_rescale, + rescale_factor=rescale_factor, + device=images.device, + ) + # if/elif as we use fused rescale and normalize if both are set to True + if do_normalize: + images = self.normalize(images.to(dtype=torch.float32), image_mean, image_std) + elif do_rescale: + images = self.rescale(images, rescale_factor) + + return images + + def center_crop( + self, + image: "torch.Tensor", + size: dict[str, int], + **kwargs, + ) -> "torch.Tensor": + """ + Center crop an image to `(size["height"], size["width"])`. If the input size is smaller than `crop_size` along + any edge, the image is padded with 0's and then center cropped. + + Args: + image (`"torch.Tensor"`): + Image to center crop. + size (`Dict[str, int]`): + Size of the output image. + + Returns: + `torch.Tensor`: The center cropped image. + """ + if size.height is None or size.width is None: + raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}") + return F.center_crop(image, (size["height"], size["width"])) + + def convert_to_rgb( + self, + image: ImageInput, + ) -> ImageInput: + """ + Converts an image to RGB format. Only converts if the image is of type PIL.Image.Image, otherwise returns the image + as is. + Args: + image (ImageInput): + The image to convert. + + Returns: + ImageInput: The converted image. + """ + return convert_to_rgb(image) + + def filter_out_unused_kwargs(self, kwargs: dict): + """ + Filter out the unused kwargs from the kwargs dictionary. + """ + if self.unused_kwargs is None: + return kwargs + + for kwarg_name in self.unused_kwargs: + if kwarg_name in kwargs: + logger.warning_once(f"This processor does not use the `{kwarg_name}` parameter. It will be ignored.") + kwargs.pop(kwarg_name) + return kwargs + + def _prepare_images_structure( + self, + images: ImageInput, + ) -> ImageInput: + """ + Prepare the images structure for processing. + + Args: + images (`ImageInput`): + The input images to process. + + Returns: + `ImageInput`: The images with a valid nesting. + """ + return make_flat_list_of_images(images) + + def _process_image( + self, + image: ImageInput, + do_convert_rgb: Optional[bool] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + device: Optional["torch.device"] = None, + ) -> "torch.Tensor": + image_type = get_image_type(image) + if image_type not in [ImageType.PIL, ImageType.TORCH, ImageType.NUMPY]: + raise ValueError(f"Unsupported input image type {image_type}") + + if do_convert_rgb: + image = self.convert_to_rgb(image) + + if image_type == ImageType.PIL: + image = F.pil_to_tensor(image) + elif image_type == ImageType.NUMPY: + # not using F.to_tensor as it doesn't handle (C, H, W) numpy arrays + image = torch.from_numpy(image).contiguous() + + # Infer the channel dimension format if not provided + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image) + + if input_data_format == ChannelDimension.LAST: + # We force the channel dimension to be first for torch tensors as this is what torchvision expects. + image = image.permute(2, 0, 1).contiguous() + + # Now that we have torch tensors, we can move them to the right device + if device is not None: + image = image.to(device) + + return image + + def _prepare_input_images( + self, + images: ImageInput, + do_convert_rgb: Optional[bool] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + device: Optional["torch.device"] = None, + ) -> list["torch.Tensor"]: + """ + Prepare the input images for processing. + """ + images = self._prepare_images_structure(images) + process_image_fn = partial( + self._process_image, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + device=device, + ) + # todo: yoni - check if we can parallelize this efficiently + processed_images = [] + for image in images: + processed_images.append(process_image_fn(image)) + + return processed_images + + def _further_process_kwargs( + self, + size: Optional[SizeDict] = None, + crop_size: Optional[SizeDict] = None, + default_to_square: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + data_format: Optional[ChannelDimension] = None, + **kwargs, + ) -> dict: + """ + Update kwargs that need further processing before being validated + Can be overridden by subclasses to customize the processing of kwargs. + """ + if kwargs is None: + kwargs = {} + if size is not None: + size = SizeDict(**get_size_dict(size=size, default_to_square=default_to_square)) + if crop_size is not None: + crop_size = SizeDict(**get_size_dict(crop_size, param_name="crop_size")) + if isinstance(image_mean, list): + image_mean = tuple(image_mean) + if isinstance(image_std, list): + image_std = tuple(image_std) + if data_format is None: + data_format = ChannelDimension.FIRST + + kwargs["size"] = size + kwargs["crop_size"] = crop_size + kwargs["default_to_square"] = default_to_square + kwargs["image_mean"] = image_mean + kwargs["image_std"] = image_std + kwargs["data_format"] = data_format + + return kwargs + + def _validate_preprocess_kwargs( + self, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, tuple[float]]] = None, + image_std: Optional[Union[float, tuple[float]]] = None, + do_resize: Optional[bool] = None, + size: Optional[SizeDict] = None, + do_center_crop: Optional[bool] = None, + crop_size: Optional[SizeDict] = None, + resample: Optional[Union["PILImageResampling", "F.InterpolationMode"]] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = None, + **kwargs, + ): + """ + validate the kwargs for the preprocess method. + """ + validate_fast_preprocess_arguments( + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + do_resize=do_resize, + size=size, + do_center_crop=do_center_crop, + crop_size=crop_size, + resample=resample, + return_tensors=return_tensors, + data_format=data_format, + ) + + @add_start_docstrings(BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS) + def preprocess(self, images: ImageInput, **kwargs: Unpack[DefaultFastImageProcessorKwargs]) -> BatchFeature: + validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self.valid_kwargs.__annotations__.keys()) + # Set default kwargs from self. This ensures that if a kwarg is not provided + # by the user, it gets its default value from the instance, or is set to None. + for kwarg_name in self.valid_kwargs.__annotations__: + kwargs.setdefault(kwarg_name, getattr(self, kwarg_name, None)) + + # Extract parameters that are only used for preparing the input images + do_convert_rgb = kwargs.pop("do_convert_rgb") + input_data_format = kwargs.pop("input_data_format") + device = kwargs.pop("device") + # Prepare input images + images = self._prepare_input_images( + images=images, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device + ) + + # Update kwargs that need further processing before being validated + kwargs = self._further_process_kwargs(**kwargs) + + # Validate kwargs + self._validate_preprocess_kwargs(**kwargs) + + # torch resize uses interpolation instead of resample + resample = kwargs.pop("resample") + kwargs["interpolation"] = ( + pil_torch_interpolation_mapping[resample] if isinstance(resample, (PILImageResampling, int)) else resample + ) + + # Pop kwargs that are not needed in _preprocess + kwargs.pop("default_to_square") + kwargs.pop("data_format") + + return self._preprocess(images=images, **kwargs) + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + interpolation: Optional["F.InterpolationMode"], + do_center_crop: bool, + crop_size: SizeDict, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: Optional[Union[float, list[float]]], + image_std: Optional[Union[float, list[float]]], + return_tensors: Optional[Union[str, TensorType]], + **kwargs, + ) -> BatchFeature: + # Group images by size for batched resizing + grouped_images, grouped_images_index = group_images_by_shape(images) + resized_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_resize: + stacked_images = self.resize(image=stacked_images, size=size, interpolation=interpolation) + resized_images_grouped[shape] = stacked_images + resized_images = reorder_images(resized_images_grouped, grouped_images_index) + + # Group images by size for further processing + # Needed in case do_resize is False, or resize returns images with different sizes + grouped_images, grouped_images_index = group_images_by_shape(resized_images) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + if do_center_crop: + stacked_images = self.center_crop(stacked_images, crop_size) + # Fused rescale and normalize + stacked_images = self.rescale_and_normalize( + stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + processed_images_grouped[shape] = stacked_images + + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images + + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + + def to_dict(self): + encoder_dict = super().to_dict() + encoder_dict.pop("_valid_processor_keys", None) + return encoder_dict + + +class SemanticSegmentationMixin: + def post_process_semantic_segmentation(self, outputs, target_sizes: list[tuple] = None): + """ + Converts the output of [`MobileNetV2ForSemanticSegmentation`] into semantic segmentation maps. Only supports PyTorch. + + Args: + outputs ([`MobileNetV2ForSemanticSegmentation`]): + Raw outputs of the model. + target_sizes (`List[Tuple]` of length `batch_size`, *optional*): + List of tuples corresponding to the requested final size (height, width) of each prediction. If unset, + predictions will not be resized. + + Returns: + semantic_segmentation: `List[torch.Tensor]` of length `batch_size`, where each item is a semantic + segmentation map of shape (height, width) corresponding to the target_sizes entry (if `target_sizes` is + specified). Each entry of each `torch.Tensor` correspond to a semantic class id. + """ + logits = outputs.logits + + # Resize logits and compute semantic segmentation maps + if target_sizes is not None: + if len(logits) != len(target_sizes): + raise ValueError( + "Make sure that you pass in as many target sizes as the batch dimension of the logits" + ) + + # if is_torch_tensor(target_sizes): + # target_sizes = target_sizes.numpy() + + semantic_segmentation = [] + + for idx in range(len(logits)): + resized_logits = torch.nn.functional.interpolate( + logits[idx].unsqueeze(dim=0), size=target_sizes[idx], mode="bilinear", align_corners=False + ) + semantic_map = resized_logits[0].argmax(dim=0) + semantic_segmentation.append(semantic_map) + else: + semantic_segmentation = logits.argmax(dim=1) + semantic_segmentation = [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])] + + return semantic_segmentation diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_transforms.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..5b0ba3f9122f17c4355c94f471af5720bde2d814 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_transforms.py @@ -0,0 +1,881 @@ +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Collection, Iterable +from math import ceil +from typing import Optional, Union + +import numpy as np + +from .image_utils import ( + ChannelDimension, + ImageInput, + get_channel_dimension_axis, + get_image_size, + infer_channel_dimension_format, +) +from .utils import ExplicitEnum, TensorType, is_jax_tensor, is_tf_tensor, is_torch_tensor +from .utils.import_utils import ( + is_flax_available, + is_tf_available, + is_torch_available, + is_vision_available, + requires_backends, +) + + +if is_vision_available(): + import PIL + + from .image_utils import PILImageResampling + +if is_torch_available(): + import torch + +if is_tf_available(): + import tensorflow as tf + +if is_flax_available(): + import jax.numpy as jnp + + +def to_channel_dimension_format( + image: np.ndarray, + channel_dim: Union[ChannelDimension, str], + input_channel_dim: Optional[Union[ChannelDimension, str]] = None, +) -> np.ndarray: + """ + Converts `image` to the channel dimension format specified by `channel_dim`. + + Args: + image (`numpy.ndarray`): + The image to have its channel dimension set. + channel_dim (`ChannelDimension`): + The channel dimension format to use. + input_channel_dim (`ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input image. + + Returns: + `np.ndarray`: The image with the channel dimension set to `channel_dim`. + """ + if not isinstance(image, np.ndarray): + raise TypeError(f"Input image must be of type np.ndarray, got {type(image)}") + + if input_channel_dim is None: + input_channel_dim = infer_channel_dimension_format(image) + + target_channel_dim = ChannelDimension(channel_dim) + if input_channel_dim == target_channel_dim: + return image + + if target_channel_dim == ChannelDimension.FIRST: + image = image.transpose((2, 0, 1)) + elif target_channel_dim == ChannelDimension.LAST: + image = image.transpose((1, 2, 0)) + else: + raise ValueError(f"Unsupported channel dimension format: {channel_dim}") + + return image + + +def rescale( + image: np.ndarray, + scale: float, + data_format: Optional[ChannelDimension] = None, + dtype: np.dtype = np.float32, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> np.ndarray: + """ + Rescales `image` by `scale`. + + Args: + image (`np.ndarray`): + The image to rescale. + scale (`float`): + The scale to use for rescaling the image. + data_format (`ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + dtype (`np.dtype`, *optional*, defaults to `np.float32`): + The dtype of the output image. Defaults to `np.float32`. Used for backwards compatibility with feature + extractors. + input_data_format (`ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input image. + + Returns: + `np.ndarray`: The rescaled image. + """ + if not isinstance(image, np.ndarray): + raise TypeError(f"Input image must be of type np.ndarray, got {type(image)}") + + rescaled_image = image.astype(np.float64) * scale # Numpy type promotion has changed, so always upcast first + if data_format is not None: + rescaled_image = to_channel_dimension_format(rescaled_image, data_format, input_data_format) + + rescaled_image = rescaled_image.astype(dtype) # Finally downcast to the desired dtype at the end + + return rescaled_image + + +def _rescale_for_pil_conversion(image): + """ + Detects whether or not the image needs to be rescaled before being converted to a PIL image. + + The assumption is that if the image is of type `np.float` and all values are between 0 and 1, it needs to be + rescaled. + """ + if image.dtype == np.uint8: + do_rescale = False + elif np.allclose(image, image.astype(int)): + if np.all(0 <= image) and np.all(image <= 255): + do_rescale = False + else: + raise ValueError( + "The image to be converted to a PIL image contains values outside the range [0, 255], " + f"got [{image.min()}, {image.max()}] which cannot be converted to uint8." + ) + elif np.all(0 <= image) and np.all(image <= 1): + do_rescale = True + else: + raise ValueError( + "The image to be converted to a PIL image contains values outside the range [0, 1], " + f"got [{image.min()}, {image.max()}] which cannot be converted to uint8." + ) + return do_rescale + + +def to_pil_image( + image: Union[np.ndarray, "PIL.Image.Image", "torch.Tensor", "tf.Tensor", "jnp.ndarray"], + do_rescale: Optional[bool] = None, + image_mode: Optional[str] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> "PIL.Image.Image": + """ + Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last axis if + needed. + + Args: + image (`PIL.Image.Image` or `numpy.ndarray` or `torch.Tensor` or `tf.Tensor`): + The image to convert to the `PIL.Image` format. + do_rescale (`bool`, *optional*): + Whether or not to apply the scaling factor (to make pixel values integers between 0 and 255). Will default + to `True` if the image type is a floating type and casting to `int` would result in a loss of precision, + and `False` otherwise. + image_mode (`str`, *optional*): + The mode to use for the PIL image. If unset, will use the default mode for the input image type. + input_data_format (`ChannelDimension`, *optional*): + The channel dimension format of the input image. If unset, will use the inferred format from the input. + + Returns: + `PIL.Image.Image`: The converted image. + """ + requires_backends(to_pil_image, ["vision"]) + + if isinstance(image, PIL.Image.Image): + return image + + # Convert all tensors to numpy arrays before converting to PIL image + if is_torch_tensor(image) or is_tf_tensor(image): + image = image.numpy() + elif is_jax_tensor(image): + image = np.array(image) + elif not isinstance(image, np.ndarray): + raise ValueError(f"Input image type not supported: {type(image)}") + + # If the channel has been moved to first dim, we put it back at the end. + image = to_channel_dimension_format(image, ChannelDimension.LAST, input_data_format) + + # If there is a single channel, we squeeze it, as otherwise PIL can't handle it. + image = np.squeeze(image, axis=-1) if image.shape[-1] == 1 else image + + # PIL.Image can only store uint8 values so we rescale the image to be between 0 and 255 if needed. + do_rescale = _rescale_for_pil_conversion(image) if do_rescale is None else do_rescale + + if do_rescale: + image = rescale(image, 255) + + image = image.astype(np.uint8) + return PIL.Image.fromarray(image, mode=image_mode) + + +def get_size_with_aspect_ratio(image_size, size, max_size=None) -> tuple[int, int]: + """ + Computes the output image size given the input image size and the desired output size. + + Args: + image_size (`Tuple[int, int]`): + The input image size. + size (`int`): + The desired output size. + max_size (`int`, *optional*): + The maximum allowed output size. + """ + height, width = image_size + raw_size = None + if max_size is not None: + min_original_size = float(min((height, width))) + max_original_size = float(max((height, width))) + if max_original_size / min_original_size * size > max_size: + raw_size = max_size * min_original_size / max_original_size + size = int(round(raw_size)) + + if (height <= width and height == size) or (width <= height and width == size): + oh, ow = height, width + elif width < height: + ow = size + if max_size is not None and raw_size is not None: + oh = int(raw_size * height / width) + else: + oh = int(size * height / width) + else: + oh = size + if max_size is not None and raw_size is not None: + ow = int(raw_size * width / height) + else: + ow = int(size * width / height) + + return (oh, ow) + + +# Logic adapted from torchvision resizing logic: https://github.com/pytorch/vision/blob/511924c1ced4ce0461197e5caa64ce5b9e558aab/torchvision/transforms/functional.py#L366 +def get_resize_output_image_size( + input_image: np.ndarray, + size: Union[int, tuple[int, int], list[int], tuple[int]], + default_to_square: bool = True, + max_size: Optional[int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> tuple: + """ + Find the target (height, width) dimension of the output image after resizing given the input image and the desired + size. + + Args: + input_image (`np.ndarray`): + The image to resize. + size (`int` or `Tuple[int, int]` or List[int] or `Tuple[int]`): + The size to use for resizing the image. If `size` is a sequence like (h, w), output size will be matched to + this. + + If `size` is an int and `default_to_square` is `True`, then image will be resized to (size, size). If + `size` is an int and `default_to_square` is `False`, then smaller edge of the image will be matched to this + number. i.e, if height > width, then image will be rescaled to (size * height / width, size). + default_to_square (`bool`, *optional*, defaults to `True`): + How to convert `size` when it is a single int. If set to `True`, the `size` will be converted to a square + (`size`,`size`). If set to `False`, will replicate + [`torchvision.transforms.Resize`](https://pytorch.org/vision/stable/transforms.html#torchvision.transforms.Resize) + with support for resizing only the smallest edge and providing an optional `max_size`. + max_size (`int`, *optional*): + The maximum allowed for the longer edge of the resized image: if the longer edge of the image is greater + than `max_size` after being resized according to `size`, then the image is resized again so that the longer + edge is equal to `max_size`. As a result, `size` might be overruled, i.e the smaller edge may be shorter + than `size`. Only used if `default_to_square` is `False`. + input_data_format (`ChannelDimension`, *optional*): + The channel dimension format of the input image. If unset, will use the inferred format from the input. + + Returns: + `tuple`: The target (height, width) dimension of the output image after resizing. + """ + if isinstance(size, (tuple, list)): + if len(size) == 2: + return tuple(size) + elif len(size) == 1: + # Perform same logic as if size was an int + size = size[0] + else: + raise ValueError("size must have 1 or 2 elements if it is a list or tuple") + + if default_to_square: + return (size, size) + + height, width = get_image_size(input_image, input_data_format) + short, long = (width, height) if width <= height else (height, width) + requested_new_short = size + + new_short, new_long = requested_new_short, int(requested_new_short * long / short) + + if max_size is not None: + if max_size <= requested_new_short: + raise ValueError( + f"max_size = {max_size} must be strictly greater than the requested " + f"size for the smaller edge size = {size}" + ) + if new_long > max_size: + new_short, new_long = int(max_size * new_short / new_long), max_size + + return (new_long, new_short) if width <= height else (new_short, new_long) + + +def resize( + image: np.ndarray, + size: tuple[int, int], + resample: "PILImageResampling" = None, + reducing_gap: Optional[int] = None, + data_format: Optional[ChannelDimension] = None, + return_numpy: bool = True, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> np.ndarray: + """ + Resizes `image` to `(height, width)` specified by `size` using the PIL library. + + Args: + image (`np.ndarray`): + The image to resize. + size (`Tuple[int, int]`): + The size to use for resizing the image. + resample (`int`, *optional*, defaults to `PILImageResampling.BILINEAR`): + The filter to user for resampling. + reducing_gap (`int`, *optional*): + Apply optimization by resizing the image in two steps. The bigger `reducing_gap`, the closer the result to + the fair resampling. See corresponding Pillow documentation for more details. + data_format (`ChannelDimension`, *optional*): + The channel dimension format of the output image. If unset, will use the inferred format from the input. + return_numpy (`bool`, *optional*, defaults to `True`): + Whether or not to return the resized image as a numpy array. If False a `PIL.Image.Image` object is + returned. + input_data_format (`ChannelDimension`, *optional*): + The channel dimension format of the input image. If unset, will use the inferred format from the input. + + Returns: + `np.ndarray`: The resized image. + """ + requires_backends(resize, ["vision"]) + + resample = resample if resample is not None else PILImageResampling.BILINEAR + + if not len(size) == 2: + raise ValueError("size must have 2 elements") + + # For all transformations, we want to keep the same data format as the input image unless otherwise specified. + # The resized image from PIL will always have channels last, so find the input format first. + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image) + data_format = input_data_format if data_format is None else data_format + + # To maintain backwards compatibility with the resizing done in previous image feature extractors, we use + # the pillow library to resize the image and then convert back to numpy + do_rescale = False + if not isinstance(image, PIL.Image.Image): + do_rescale = _rescale_for_pil_conversion(image) + image = to_pil_image(image, do_rescale=do_rescale, input_data_format=input_data_format) + height, width = size + # PIL images are in the format (width, height) + resized_image = image.resize((width, height), resample=resample, reducing_gap=reducing_gap) + + if return_numpy: + resized_image = np.array(resized_image) + # If the input image channel dimension was of size 1, then it is dropped when converting to a PIL image + # so we need to add it back if necessary. + resized_image = np.expand_dims(resized_image, axis=-1) if resized_image.ndim == 2 else resized_image + # The image is always in channels last format after converting from a PIL image + resized_image = to_channel_dimension_format( + resized_image, data_format, input_channel_dim=ChannelDimension.LAST + ) + # If an image was rescaled to be in the range [0, 255] before converting to a PIL image, then we need to + # rescale it back to the original range. + resized_image = rescale(resized_image, 1 / 255) if do_rescale else resized_image + return resized_image + + +def normalize( + image: np.ndarray, + mean: Union[float, Collection[float]], + std: Union[float, Collection[float]], + data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> np.ndarray: + """ + Normalizes `image` using the mean and standard deviation specified by `mean` and `std`. + + image = (image - mean) / std + + Args: + image (`np.ndarray`): + The image to normalize. + mean (`float` or `Collection[float]`): + The mean to use for normalization. + std (`float` or `Collection[float]`): + The standard deviation to use for normalization. + data_format (`ChannelDimension`, *optional*): + The channel dimension format of the output image. If unset, will use the inferred format from the input. + input_data_format (`ChannelDimension`, *optional*): + The channel dimension format of the input image. If unset, will use the inferred format from the input. + """ + if not isinstance(image, np.ndarray): + raise ValueError("image must be a numpy array") + + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image) + + channel_axis = get_channel_dimension_axis(image, input_data_format=input_data_format) + num_channels = image.shape[channel_axis] + + # We cast to float32 to avoid errors that can occur when subtracting uint8 values. + # We preserve the original dtype if it is a float type to prevent upcasting float16. + if not np.issubdtype(image.dtype, np.floating): + image = image.astype(np.float32) + + if isinstance(mean, Collection): + if len(mean) != num_channels: + raise ValueError(f"mean must have {num_channels} elements if it is an iterable, got {len(mean)}") + else: + mean = [mean] * num_channels + mean = np.array(mean, dtype=image.dtype) + + if isinstance(std, Collection): + if len(std) != num_channels: + raise ValueError(f"std must have {num_channels} elements if it is an iterable, got {len(std)}") + else: + std = [std] * num_channels + std = np.array(std, dtype=image.dtype) + + if input_data_format == ChannelDimension.LAST: + image = (image - mean) / std + else: + image = ((image.T - mean) / std).T + + image = to_channel_dimension_format(image, data_format, input_data_format) if data_format is not None else image + return image + + +def center_crop( + image: np.ndarray, + size: tuple[int, int], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> np.ndarray: + """ + Crops the `image` to the specified `size` using a center crop. Note that if the image is too small to be cropped to + the size given, it will be padded (so the returned result will always be of size `size`). + + Args: + image (`np.ndarray`): + The image to crop. + size (`Tuple[int, int]`): + The target size for the cropped image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + If unset, will use the inferred format of the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + If unset, will use the inferred format of the input image. + Returns: + `np.ndarray`: The cropped image. + """ + requires_backends(center_crop, ["vision"]) + + if not isinstance(image, np.ndarray): + raise TypeError(f"Input image must be of type np.ndarray, got {type(image)}") + + if not isinstance(size, Iterable) or len(size) != 2: + raise ValueError("size must have 2 elements representing the height and width of the output image") + + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image) + output_data_format = data_format if data_format is not None else input_data_format + + # We perform the crop in (C, H, W) format and then convert to the output format + image = to_channel_dimension_format(image, ChannelDimension.FIRST, input_data_format) + + orig_height, orig_width = get_image_size(image, ChannelDimension.FIRST) + crop_height, crop_width = size + crop_height, crop_width = int(crop_height), int(crop_width) + + # In case size is odd, (image_shape[0] + size[0]) // 2 won't give the proper result. + top = (orig_height - crop_height) // 2 + bottom = top + crop_height + # In case size is odd, (image_shape[1] + size[1]) // 2 won't give the proper result. + left = (orig_width - crop_width) // 2 + right = left + crop_width + + # Check if cropped area is within image boundaries + if top >= 0 and bottom <= orig_height and left >= 0 and right <= orig_width: + image = image[..., top:bottom, left:right] + image = to_channel_dimension_format(image, output_data_format, ChannelDimension.FIRST) + return image + + # Otherwise, we may need to pad if the image is too small. Oh joy... + new_height = max(crop_height, orig_height) + new_width = max(crop_width, orig_width) + new_shape = image.shape[:-2] + (new_height, new_width) + new_image = np.zeros_like(image, shape=new_shape) + + # If the image is too small, pad it with zeros + top_pad = ceil((new_height - orig_height) / 2) + bottom_pad = top_pad + orig_height + left_pad = ceil((new_width - orig_width) / 2) + right_pad = left_pad + orig_width + new_image[..., top_pad:bottom_pad, left_pad:right_pad] = image + + top += top_pad + bottom += top_pad + left += left_pad + right += left_pad + + new_image = new_image[..., max(0, top) : min(new_height, bottom), max(0, left) : min(new_width, right)] + new_image = to_channel_dimension_format(new_image, output_data_format, ChannelDimension.FIRST) + + return new_image + + +def _center_to_corners_format_torch(bboxes_center: "torch.Tensor") -> "torch.Tensor": + center_x, center_y, width, height = bboxes_center.unbind(-1) + bbox_corners = torch.stack( + # top left x, top left y, bottom right x, bottom right y + [(center_x - 0.5 * width), (center_y - 0.5 * height), (center_x + 0.5 * width), (center_y + 0.5 * height)], + dim=-1, + ) + return bbox_corners + + +def _center_to_corners_format_numpy(bboxes_center: np.ndarray) -> np.ndarray: + center_x, center_y, width, height = bboxes_center.T + bboxes_corners = np.stack( + # top left x, top left y, bottom right x, bottom right y + [center_x - 0.5 * width, center_y - 0.5 * height, center_x + 0.5 * width, center_y + 0.5 * height], + axis=-1, + ) + return bboxes_corners + + +def _center_to_corners_format_tf(bboxes_center: "tf.Tensor") -> "tf.Tensor": + center_x, center_y, width, height = tf.unstack(bboxes_center, axis=-1) + bboxes_corners = tf.stack( + # top left x, top left y, bottom right x, bottom right y + [center_x - 0.5 * width, center_y - 0.5 * height, center_x + 0.5 * width, center_y + 0.5 * height], + axis=-1, + ) + return bboxes_corners + + +# 2 functions below inspired by https://github.com/facebookresearch/detr/blob/master/util/box_ops.py +def center_to_corners_format(bboxes_center: TensorType) -> TensorType: + """ + Converts bounding boxes from center format to corners format. + + center format: contains the coordinate for the center of the box and its width, height dimensions + (center_x, center_y, width, height) + corners format: contains the coordinates for the top-left and bottom-right corners of the box + (top_left_x, top_left_y, bottom_right_x, bottom_right_y) + """ + # Function is used during model forward pass, so we use the input framework if possible, without + # converting to numpy + if is_torch_tensor(bboxes_center): + return _center_to_corners_format_torch(bboxes_center) + elif isinstance(bboxes_center, np.ndarray): + return _center_to_corners_format_numpy(bboxes_center) + elif is_tf_tensor(bboxes_center): + return _center_to_corners_format_tf(bboxes_center) + + raise ValueError(f"Unsupported input type {type(bboxes_center)}") + + +def _corners_to_center_format_torch(bboxes_corners: "torch.Tensor") -> "torch.Tensor": + top_left_x, top_left_y, bottom_right_x, bottom_right_y = bboxes_corners.unbind(-1) + b = [ + (top_left_x + bottom_right_x) / 2, # center x + (top_left_y + bottom_right_y) / 2, # center y + (bottom_right_x - top_left_x), # width + (bottom_right_y - top_left_y), # height + ] + return torch.stack(b, dim=-1) + + +def _corners_to_center_format_numpy(bboxes_corners: np.ndarray) -> np.ndarray: + top_left_x, top_left_y, bottom_right_x, bottom_right_y = bboxes_corners.T + bboxes_center = np.stack( + [ + (top_left_x + bottom_right_x) / 2, # center x + (top_left_y + bottom_right_y) / 2, # center y + (bottom_right_x - top_left_x), # width + (bottom_right_y - top_left_y), # height + ], + axis=-1, + ) + return bboxes_center + + +def _corners_to_center_format_tf(bboxes_corners: "tf.Tensor") -> "tf.Tensor": + top_left_x, top_left_y, bottom_right_x, bottom_right_y = tf.unstack(bboxes_corners, axis=-1) + bboxes_center = tf.stack( + [ + (top_left_x + bottom_right_x) / 2, # center x + (top_left_y + bottom_right_y) / 2, # center y + (bottom_right_x - top_left_x), # width + (bottom_right_y - top_left_y), # height + ], + axis=-1, + ) + return bboxes_center + + +def corners_to_center_format(bboxes_corners: TensorType) -> TensorType: + """ + Converts bounding boxes from corners format to center format. + + corners format: contains the coordinates for the top-left and bottom-right corners of the box + (top_left_x, top_left_y, bottom_right_x, bottom_right_y) + center format: contains the coordinate for the center of the box and its the width, height dimensions + (center_x, center_y, width, height) + """ + # Inverse function accepts different input types so implemented here too + if is_torch_tensor(bboxes_corners): + return _corners_to_center_format_torch(bboxes_corners) + elif isinstance(bboxes_corners, np.ndarray): + return _corners_to_center_format_numpy(bboxes_corners) + elif is_tf_tensor(bboxes_corners): + return _corners_to_center_format_tf(bboxes_corners) + + raise ValueError(f"Unsupported input type {type(bboxes_corners)}") + + +# 2 functions below copied from https://github.com/cocodataset/panopticapi/blob/master/panopticapi/utils.py +# Copyright (c) 2018, Alexander Kirillov +# All rights reserved. +def rgb_to_id(color): + """ + Converts RGB color to unique ID. + """ + if isinstance(color, np.ndarray) and len(color.shape) == 3: + if color.dtype == np.uint8: + color = color.astype(np.int32) + return color[:, :, 0] + 256 * color[:, :, 1] + 256 * 256 * color[:, :, 2] + return int(color[0] + 256 * color[1] + 256 * 256 * color[2]) + + +def id_to_rgb(id_map): + """ + Converts unique ID to RGB color. + """ + if isinstance(id_map, np.ndarray): + id_map_copy = id_map.copy() + rgb_shape = tuple(list(id_map.shape) + [3]) + rgb_map = np.zeros(rgb_shape, dtype=np.uint8) + for i in range(3): + rgb_map[..., i] = id_map_copy % 256 + id_map_copy //= 256 + return rgb_map + color = [] + for _ in range(3): + color.append(id_map % 256) + id_map //= 256 + return color + + +class PaddingMode(ExplicitEnum): + """ + Enum class for the different padding modes to use when padding images. + """ + + CONSTANT = "constant" + REFLECT = "reflect" + REPLICATE = "replicate" + SYMMETRIC = "symmetric" + + +def pad( + image: np.ndarray, + padding: Union[int, tuple[int, int], Iterable[tuple[int, int]]], + mode: PaddingMode = PaddingMode.CONSTANT, + constant_values: Union[float, Iterable[float]] = 0.0, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> np.ndarray: + """ + Pads the `image` with the specified (height, width) `padding` and `mode`. + + Args: + image (`np.ndarray`): + The image to pad. + padding (`int` or `Tuple[int, int]` or `Iterable[Tuple[int, int]]`): + Padding to apply to the edges of the height, width axes. Can be one of three formats: + - `((before_height, after_height), (before_width, after_width))` unique pad widths for each axis. + - `((before, after),)` yields same before and after pad for height and width. + - `(pad,)` or int is a shortcut for before = after = pad width for all axes. + mode (`PaddingMode`): + The padding mode to use. Can be one of: + - `"constant"`: pads with a constant value. + - `"reflect"`: pads with the reflection of the vector mirrored on the first and last values of the + vector along each axis. + - `"replicate"`: pads with the replication of the last value on the edge of the array along each axis. + - `"symmetric"`: pads with the reflection of the vector mirrored along the edge of the array. + constant_values (`float` or `Iterable[float]`, *optional*): + The value to use for the padding if `mode` is `"constant"`. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + If unset, will use same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + If unset, will use the inferred format of the input image. + + Returns: + `np.ndarray`: The padded image. + + """ + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image) + + def _expand_for_data_format(values): + """ + Convert values to be in the format expected by np.pad based on the data format. + """ + if isinstance(values, (int, float)): + values = ((values, values), (values, values)) + elif isinstance(values, tuple) and len(values) == 1: + values = ((values[0], values[0]), (values[0], values[0])) + elif isinstance(values, tuple) and len(values) == 2 and isinstance(values[0], int): + values = (values, values) + elif isinstance(values, tuple) and len(values) == 2 and isinstance(values[0], tuple): + values = values + else: + raise ValueError(f"Unsupported format: {values}") + + # add 0 for channel dimension + values = ((0, 0), *values) if input_data_format == ChannelDimension.FIRST else (*values, (0, 0)) + + # Add additional padding if there's a batch dimension + values = (0, *values) if image.ndim == 4 else values + return values + + padding = _expand_for_data_format(padding) + + if mode == PaddingMode.CONSTANT: + constant_values = _expand_for_data_format(constant_values) + image = np.pad(image, padding, mode="constant", constant_values=constant_values) + elif mode == PaddingMode.REFLECT: + image = np.pad(image, padding, mode="reflect") + elif mode == PaddingMode.REPLICATE: + image = np.pad(image, padding, mode="edge") + elif mode == PaddingMode.SYMMETRIC: + image = np.pad(image, padding, mode="symmetric") + else: + raise ValueError(f"Invalid padding mode: {mode}") + + image = to_channel_dimension_format(image, data_format, input_data_format) if data_format is not None else image + return image + + +# TODO (Amy): Accept 1/3/4 channel numpy array as input and return np.array as default +def convert_to_rgb(image: ImageInput) -> ImageInput: + """ + Converts an image to RGB format. Only converts if the image is of type PIL.Image.Image, otherwise returns the image + as is. + Args: + image (Image): + The image to convert. + """ + requires_backends(convert_to_rgb, ["vision"]) + + if not isinstance(image, PIL.Image.Image): + return image + + if image.mode == "RGB": + return image + + image = image.convert("RGB") + return image + + +def flip_channel_order( + image: np.ndarray, + data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +) -> np.ndarray: + """ + Flips the channel order of the image. + + If the image is in RGB format, it will be converted to BGR and vice versa. + + Args: + image (`np.ndarray`): + The image to flip. + data_format (`ChannelDimension`, *optional*): + The channel dimension format for the output image. Can be one of: + - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + If unset, will use same as the input image. + input_data_format (`ChannelDimension`, *optional*): + The channel dimension format for the input image. Can be one of: + - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + If unset, will use the inferred format of the input image. + """ + input_data_format = infer_channel_dimension_format(image) if input_data_format is None else input_data_format + + if input_data_format == ChannelDimension.LAST: + image = image[..., ::-1] + elif input_data_format == ChannelDimension.FIRST: + image = image[::-1, ...] + else: + raise ValueError(f"Unsupported channel dimension: {input_data_format}") + + if data_format is not None: + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + return image + + +def _cast_tensor_to_float(x): + if x.is_floating_point(): + return x + return x.float() + + +def group_images_by_shape( + images: list["torch.Tensor"], +) -> tuple[dict[tuple[int, int], list["torch.Tensor"]], dict[int, tuple[tuple[int, int], int]]]: + """ + Groups images by shape. + Returns a dictionary with the shape as key and a list of images with that shape as value, + and a dictionary with the index of the image in the original list as key and the shape and index in the grouped list as value. + """ + grouped_images = {} + grouped_images_index = {} + for i, image in enumerate(images): + shape = image.shape[1:] + if shape not in grouped_images: + grouped_images[shape] = [] + grouped_images[shape].append(image) + grouped_images_index[i] = (shape, len(grouped_images[shape]) - 1) + # stack images with the same shape + grouped_images = {shape: torch.stack(images, dim=0) for shape, images in grouped_images.items()} + return grouped_images, grouped_images_index + + +def reorder_images( + processed_images: dict[tuple[int, int], "torch.Tensor"], grouped_images_index: dict[int, tuple[int, int]] +) -> list["torch.Tensor"]: + """ + Reconstructs a list of images in the original order. + """ + return [ + processed_images[grouped_images_index[i][0]][grouped_images_index[i][1]] + for i in range(len(grouped_images_index)) + ] + + +class NumpyToTensor: + """ + Convert a numpy array to a PyTorch tensor. + """ + + def __call__(self, image: np.ndarray): + # Same as in PyTorch, we assume incoming numpy images are in HWC format + # c.f. https://github.com/pytorch/vision/blob/61d97f41bc209e1407dcfbd685d2ee2da9c1cdad/torchvision/transforms/functional.py#L154 + return torch.from_numpy(image.transpose(2, 0, 1)).contiguous() diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f07ac1ae7d91dea63514f74ee86a46051383d0ef --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/image_utils.py @@ -0,0 +1,1345 @@ +# Copyright 2021 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +from collections.abc import Iterable +from contextlib import redirect_stdout +from dataclasses import dataclass +from io import BytesIO +from typing import TYPE_CHECKING, Callable, Optional, Union + +import numpy as np +import requests +from packaging import version + +from .utils import ( + ExplicitEnum, + is_av_available, + is_cv2_available, + is_decord_available, + is_jax_tensor, + is_numpy_array, + is_tf_tensor, + is_torch_available, + is_torch_tensor, + is_torchvision_available, + is_vision_available, + is_yt_dlp_available, + logging, + requires_backends, + to_numpy, +) +from .utils.constants import ( # noqa: F401 + IMAGENET_DEFAULT_MEAN, + IMAGENET_DEFAULT_STD, + IMAGENET_STANDARD_MEAN, + IMAGENET_STANDARD_STD, + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, +) + + +if is_vision_available(): + import PIL.Image + import PIL.ImageOps + + if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): + PILImageResampling = PIL.Image.Resampling + else: + PILImageResampling = PIL.Image + + if is_torchvision_available(): + from torchvision import io as torchvision_io + from torchvision.transforms import InterpolationMode + + pil_torch_interpolation_mapping = { + PILImageResampling.NEAREST: InterpolationMode.NEAREST, + PILImageResampling.BOX: InterpolationMode.BOX, + PILImageResampling.BILINEAR: InterpolationMode.BILINEAR, + PILImageResampling.HAMMING: InterpolationMode.HAMMING, + PILImageResampling.BICUBIC: InterpolationMode.BICUBIC, + PILImageResampling.LANCZOS: InterpolationMode.LANCZOS, + } + + +if TYPE_CHECKING: + if is_torch_available(): + import torch + + +logger = logging.get_logger(__name__) + + +ImageInput = Union[ + "PIL.Image.Image", np.ndarray, "torch.Tensor", list["PIL.Image.Image"], list[np.ndarray], list["torch.Tensor"] +] # noqa + + +VideoInput = Union[ + list["PIL.Image.Image"], + "np.ndarray", + "torch.Tensor", + list["np.ndarray"], + list["torch.Tensor"], + list[list["PIL.Image.Image"]], + list[list["np.ndarray"]], + list[list["torch.Tensor"]], +] # noqa + + +class ChannelDimension(ExplicitEnum): + FIRST = "channels_first" + LAST = "channels_last" + + +class AnnotationFormat(ExplicitEnum): + COCO_DETECTION = "coco_detection" + COCO_PANOPTIC = "coco_panoptic" + + +class AnnotionFormat(ExplicitEnum): + COCO_DETECTION = AnnotationFormat.COCO_DETECTION.value + COCO_PANOPTIC = AnnotationFormat.COCO_PANOPTIC.value + + +@dataclass +class VideoMetadata: + total_num_frames: int + fps: float + duration: float + video_backend: str + + +AnnotationType = dict[str, Union[int, str, list[dict]]] + + +def is_pil_image(img): + return is_vision_available() and isinstance(img, PIL.Image.Image) + + +class ImageType(ExplicitEnum): + PIL = "pillow" + TORCH = "torch" + NUMPY = "numpy" + TENSORFLOW = "tensorflow" + JAX = "jax" + + +def get_image_type(image): + if is_pil_image(image): + return ImageType.PIL + if is_torch_tensor(image): + return ImageType.TORCH + if is_numpy_array(image): + return ImageType.NUMPY + if is_tf_tensor(image): + return ImageType.TENSORFLOW + if is_jax_tensor(image): + return ImageType.JAX + raise ValueError(f"Unrecognised image type {type(image)}") + + +def is_valid_image(img): + return is_pil_image(img) or is_numpy_array(img) or is_torch_tensor(img) or is_tf_tensor(img) or is_jax_tensor(img) + + +def is_valid_list_of_images(images: list): + return images and all(is_valid_image(image) for image in images) + + +def valid_images(imgs): + # If we have an list of images, make sure every image is valid + if isinstance(imgs, (list, tuple)): + for img in imgs: + if not valid_images(img): + return False + # If not a list of tuple, we have been given a single image or batched tensor of images + elif not is_valid_image(imgs): + return False + return True + + +def is_batched(img): + if isinstance(img, (list, tuple)): + return is_valid_image(img[0]) + return False + + +def is_scaled_image(image: np.ndarray) -> bool: + """ + Checks to see whether the pixel values have already been rescaled to [0, 1]. + """ + if image.dtype == np.uint8: + return False + + # It's possible the image has pixel values in [0, 255] but is of floating type + return np.min(image) >= 0 and np.max(image) <= 1 + + +def make_list_of_images(images, expected_ndims: int = 3) -> list[ImageInput]: + """ + Ensure that the output is a list of images. If the input is a single image, it is converted to a list of length 1. + If the input is a batch of images, it is converted to a list of images. + + Args: + images (`ImageInput`): + Image of images to turn into a list of images. + expected_ndims (`int`, *optional*, defaults to 3): + Expected number of dimensions for a single input image. If the input image has a different number of + dimensions, an error is raised. + """ + if is_batched(images): + return images + + # Either the input is a single image, in which case we create a list of length 1 + if is_pil_image(images): + # PIL images are never batched + return [images] + + if is_valid_image(images): + if images.ndim == expected_ndims + 1: + # Batch of images + images = list(images) + elif images.ndim == expected_ndims: + # Single image + images = [images] + else: + raise ValueError( + f"Invalid image shape. Expected either {expected_ndims + 1} or {expected_ndims} dimensions, but got" + f" {images.ndim} dimensions." + ) + return images + raise ValueError( + "Invalid image type. Expected either PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or " + f"jax.ndarray, but got {type(images)}." + ) + + +def make_flat_list_of_images( + images: Union[list[ImageInput], ImageInput], +) -> ImageInput: + """ + Ensure that the output is a flat list of images. If the input is a single image, it is converted to a list of length 1. + If the input is a nested list of images, it is converted to a flat list of images. + Args: + images (`Union[List[ImageInput], ImageInput]`): + The input image. + Returns: + list: A list of images or a 4d array of images. + """ + # If the input is a nested list of images, we flatten it + if ( + isinstance(images, (list, tuple)) + and all(isinstance(images_i, (list, tuple)) for images_i in images) + and all(is_valid_list_of_images(images_i) for images_i in images) + ): + return [img for img_list in images for img in img_list] + + if isinstance(images, (list, tuple)) and is_valid_list_of_images(images): + if is_pil_image(images[0]) or images[0].ndim == 3: + return images + if images[0].ndim == 4: + return [img for img_list in images for img in img_list] + + if is_valid_image(images): + if is_pil_image(images) or images.ndim == 3: + return [images] + if images.ndim == 4: + return list(images) + + raise ValueError(f"Could not make a flat list of images from {images}") + + +def make_nested_list_of_images( + images: Union[list[ImageInput], ImageInput], +) -> ImageInput: + """ + Ensure that the output is a nested list of images. + Args: + images (`Union[List[ImageInput], ImageInput]`): + The input image. + Returns: + list: A list of list of images or a list of 4d array of images. + """ + # If it's a list of batches, it's already in the right format + if ( + isinstance(images, (list, tuple)) + and all(isinstance(images_i, (list, tuple)) for images_i in images) + and all(is_valid_list_of_images(images_i) for images_i in images) + ): + return images + + # If it's a list of images, it's a single batch, so convert it to a list of lists + if isinstance(images, (list, tuple)) and is_valid_list_of_images(images): + if is_pil_image(images[0]) or images[0].ndim == 3: + return [images] + if images[0].ndim == 4: + return [list(image) for image in images] + + # If it's a single image, convert it to a list of lists + if is_valid_image(images): + if is_pil_image(images) or images.ndim == 3: + return [[images]] + if images.ndim == 4: + return [list(images)] + + raise ValueError("Invalid input type. Must be a single image, a list of images, or a list of batches of images.") + + +def make_batched_videos(videos) -> VideoInput: + """ + Ensure that the input is a list of videos. + Args: + videos (`VideoInput`): + Video or videos to turn into a list of videos. + Returns: + list: A list of videos. + """ + if isinstance(videos, (list, tuple)) and isinstance(videos[0], (list, tuple)) and is_valid_image(videos[0][0]): + # case 1: nested batch of videos so we flatten it + if not is_pil_image(videos[0][0]) and videos[0][0].ndim == 4: + videos = [[video for batch_list in batched_videos for video in batch_list] for batched_videos in videos] + # case 2: list of videos represented as list of video frames + return videos + + elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]): + if is_pil_image(videos[0]) or videos[0].ndim == 3: + return [videos] + elif videos[0].ndim == 4: + return [list(video) for video in videos] + + elif is_valid_image(videos): + if is_pil_image(videos) or videos.ndim == 3: + return [[videos]] + elif videos.ndim == 4: + return [list(videos)] + + raise ValueError(f"Could not make batched video from {videos}") + + +def to_numpy_array(img) -> np.ndarray: + if not is_valid_image(img): + raise ValueError(f"Invalid image type: {type(img)}") + + if is_vision_available() and isinstance(img, PIL.Image.Image): + return np.array(img) + return to_numpy(img) + + +def infer_channel_dimension_format( + image: np.ndarray, num_channels: Optional[Union[int, tuple[int, ...]]] = None +) -> ChannelDimension: + """ + Infers the channel dimension format of `image`. + + Args: + image (`np.ndarray`): + The image to infer the channel dimension of. + num_channels (`int` or `Tuple[int, ...]`, *optional*, defaults to `(1, 3)`): + The number of channels of the image. + + Returns: + The channel dimension of the image. + """ + num_channels = num_channels if num_channels is not None else (1, 3) + num_channels = (num_channels,) if isinstance(num_channels, int) else num_channels + + if image.ndim == 3: + first_dim, last_dim = 0, 2 + elif image.ndim == 4: + first_dim, last_dim = 1, 3 + else: + raise ValueError(f"Unsupported number of image dimensions: {image.ndim}") + + if image.shape[first_dim] in num_channels and image.shape[last_dim] in num_channels: + logger.warning( + f"The channel dimension is ambiguous. Got image shape {image.shape}. Assuming channels are the first dimension." + ) + return ChannelDimension.FIRST + elif image.shape[first_dim] in num_channels: + return ChannelDimension.FIRST + elif image.shape[last_dim] in num_channels: + return ChannelDimension.LAST + raise ValueError("Unable to infer channel dimension format") + + +def get_channel_dimension_axis( + image: np.ndarray, input_data_format: Optional[Union[ChannelDimension, str]] = None +) -> int: + """ + Returns the channel dimension axis of the image. + + Args: + image (`np.ndarray`): + The image to get the channel dimension axis of. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the image. If `None`, will infer the channel dimension from the image. + + Returns: + The channel dimension axis of the image. + """ + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image) + if input_data_format == ChannelDimension.FIRST: + return image.ndim - 3 + elif input_data_format == ChannelDimension.LAST: + return image.ndim - 1 + raise ValueError(f"Unsupported data format: {input_data_format}") + + +def get_image_size(image: np.ndarray, channel_dim: ChannelDimension = None) -> tuple[int, int]: + """ + Returns the (height, width) dimensions of the image. + + Args: + image (`np.ndarray`): + The image to get the dimensions of. + channel_dim (`ChannelDimension`, *optional*): + Which dimension the channel dimension is in. If `None`, will infer the channel dimension from the image. + + Returns: + A tuple of the image's height and width. + """ + if channel_dim is None: + channel_dim = infer_channel_dimension_format(image) + + if channel_dim == ChannelDimension.FIRST: + return image.shape[-2], image.shape[-1] + elif channel_dim == ChannelDimension.LAST: + return image.shape[-3], image.shape[-2] + else: + raise ValueError(f"Unsupported data format: {channel_dim}") + + +def get_image_size_for_max_height_width( + image_size: tuple[int, int], + max_height: int, + max_width: int, +) -> tuple[int, int]: + """ + Computes the output image size given the input image and the maximum allowed height and width. Keep aspect ratio. + Important, even if image_height < max_height and image_width < max_width, the image will be resized + to at least one of the edges be equal to max_height or max_width. + + For example: + - input_size: (100, 200), max_height: 50, max_width: 50 -> output_size: (25, 50) + - input_size: (100, 200), max_height: 200, max_width: 500 -> output_size: (200, 400) + + Args: + image_size (`Tuple[int, int]`): + The image to resize. + max_height (`int`): + The maximum allowed height. + max_width (`int`): + The maximum allowed width. + """ + height, width = image_size + height_scale = max_height / height + width_scale = max_width / width + min_scale = min(height_scale, width_scale) + new_height = int(height * min_scale) + new_width = int(width * min_scale) + return new_height, new_width + + +def is_valid_annotation_coco_detection(annotation: dict[str, Union[list, tuple]]) -> bool: + if ( + isinstance(annotation, dict) + and "image_id" in annotation + and "annotations" in annotation + and isinstance(annotation["annotations"], (list, tuple)) + and ( + # an image can have no annotations + len(annotation["annotations"]) == 0 or isinstance(annotation["annotations"][0], dict) + ) + ): + return True + return False + + +def is_valid_annotation_coco_panoptic(annotation: dict[str, Union[list, tuple]]) -> bool: + if ( + isinstance(annotation, dict) + and "image_id" in annotation + and "segments_info" in annotation + and "file_name" in annotation + and isinstance(annotation["segments_info"], (list, tuple)) + and ( + # an image can have no segments + len(annotation["segments_info"]) == 0 or isinstance(annotation["segments_info"][0], dict) + ) + ): + return True + return False + + +def valid_coco_detection_annotations(annotations: Iterable[dict[str, Union[list, tuple]]]) -> bool: + return all(is_valid_annotation_coco_detection(ann) for ann in annotations) + + +def valid_coco_panoptic_annotations(annotations: Iterable[dict[str, Union[list, tuple]]]) -> bool: + return all(is_valid_annotation_coco_panoptic(ann) for ann in annotations) + + +def load_image(image: Union[str, "PIL.Image.Image"], timeout: Optional[float] = None) -> "PIL.Image.Image": + """ + Loads `image` to a PIL Image. + + Args: + image (`str` or `PIL.Image.Image`): + The image to convert to the PIL Image format. + timeout (`float`, *optional*): + The timeout value in seconds for the URL request. + + Returns: + `PIL.Image.Image`: A PIL Image. + """ + requires_backends(load_image, ["vision"]) + if isinstance(image, str): + if image.startswith("http://") or image.startswith("https://"): + # We need to actually check for a real protocol, otherwise it's impossible to use a local file + # like http_huggingface_co.png + image = PIL.Image.open(BytesIO(requests.get(image, timeout=timeout).content)) + elif os.path.isfile(image): + image = PIL.Image.open(image) + else: + if image.startswith("data:image/"): + image = image.split(",")[1] + + # Try to load as base64 + try: + b64 = base64.decodebytes(image.encode()) + image = PIL.Image.open(BytesIO(b64)) + except Exception as e: + raise ValueError( + f"Incorrect image source. Must be a valid URL starting with `http://` or `https://`, a valid path to an image file, or a base64 encoded string. Got {image}. Failed with {e}" + ) + elif isinstance(image, PIL.Image.Image): + image = image + else: + raise TypeError( + "Incorrect format used for image. Should be an url linking to an image, a base64 string, a local path, or a PIL image." + ) + image = PIL.ImageOps.exif_transpose(image) + image = image.convert("RGB") + return image + + +def default_sample_indices_fn(metadata: VideoMetadata, num_frames=None, fps=None, **kwargs): + """ + A default sampling function that replicates the logic used in get_uniform_frame_indices, + while optionally handling `fps` if `num_frames` is not provided. + + Args: + metadata (`VideoMetadata`): + `VideoMetadata` object containing metadata about the video, such as "total_num_frames" or "fps". + num_frames (`int`, *optional*): + Number of frames to sample uniformly. + fps (`int`, *optional*): + Desired frames per second. Takes priority over num_frames if both are provided. + + Returns: + `np.ndarray`: Array of frame indices to sample. + """ + total_num_frames = metadata.total_num_frames + video_fps = metadata.fps + + # If num_frames is not given but fps is, calculate num_frames from fps + if num_frames is None and fps is not None: + num_frames = int(total_num_frames / video_fps * fps) + if num_frames > total_num_frames: + raise ValueError( + f"When loading the video with fps={fps}, we computed num_frames={num_frames} " + f"which exceeds total_num_frames={total_num_frames}. Check fps or video metadata." + ) + + if num_frames is not None: + indices = np.arange(0, total_num_frames, total_num_frames / num_frames, dtype=int) + else: + indices = np.arange(0, total_num_frames, dtype=int) + return indices + + +def read_video_opencv( + video_path: str, + sample_indices_fn: Callable, + **kwargs, +): + """ + Decode a video using the OpenCV backend. + + Args: + video_path (`str`): + Path to the video file. + sample_indices_fn (`Callable`): + A callable function that will return indices at which the video should be sampled. If the video has to be loaded using + by a different sampling technique than provided by `num_frames` or `fps` arguments, one should provide their own `sample_indices_fn`. + If not provided, simple uniform sampling with fps is performed. + Example: + def sample_indices_fn(metadata, **kwargs): + return np.linspace(0, metadata.total_num_frames - 1, num_frames, dtype=int) + + Returns: + Tuple[`np.array`, `VideoMetadata`]: A tuple containing: + - Numpy array of frames in RGB (shape: [num_frames, height, width, 3]). + - `VideoMetadata` object. + """ + # Lazy import cv2 + requires_backends(read_video_opencv, ["cv2"]) + import cv2 + + video = cv2.VideoCapture(video_path) + total_num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + video_fps = video.get(cv2.CAP_PROP_FPS) + duration = total_num_frames / video_fps if video_fps else 0 + metadata = VideoMetadata( + total_num_frames=int(total_num_frames), fps=float(video_fps), duration=float(duration), video_backend="opencv" + ) + indices = sample_indices_fn(metadata=metadata, **kwargs) + + index = 0 + frames = [] + while video.isOpened(): + success, frame = video.read() + if not success: + break + if index in indices: + height, width, channel = frame.shape + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frames.append(frame[0:height, 0:width, 0:channel]) + if success: + index += 1 + if index >= total_num_frames: + break + + video.release() + metadata.frames_indices = indices + return np.stack(frames), metadata + + +def read_video_decord( + video_path: str, + sample_indices_fn: Optional[Callable] = None, + **kwargs, +): + """ + Decode a video using the Decord backend. + + Args: + video_path (`str`): + Path to the video file. + sample_indices_fn (`Callable`, *optional*): + A callable function that will return indices at which the video should be sampled. If the video has to be loaded using + by a different sampling technique than provided by `num_frames` or `fps` arguments, one should provide their own `sample_indices_fn`. + If not provided, simple uniform sampling with fps is performed. + Example: + def sample_indices_fn(metadata, **kwargs): + return np.linspace(0, metadata.total_num_frames - 1, num_frames, dtype=int) + + Returns: + Tuple[`np.array`, `VideoMetadata`]: A tuple containing: + - Numpy array of frames in RGB (shape: [num_frames, height, width, 3]). + - `VideoMetadata` object. + """ + # Lazy import from decord + requires_backends(read_video_decord, ["decord"]) + from decord import VideoReader, cpu + + vr = VideoReader(uri=video_path, ctx=cpu(0)) # decord has problems with gpu + video_fps = vr.get_avg_fps() + total_num_frames = len(vr) + duration = total_num_frames / video_fps if video_fps else 0 + metadata = VideoMetadata( + total_num_frames=int(total_num_frames), fps=float(video_fps), duration=float(duration), video_backend="decord" + ) + + indices = sample_indices_fn(metadata=metadata, **kwargs) + + frames = vr.get_batch(indices).asnumpy() + metadata.frames_indices = indices + return frames, metadata + + +def read_video_pyav( + video_path: str, + sample_indices_fn: Callable, + **kwargs, +): + """ + Decode the video with PyAV decoder. + + Args: + video_path (`str`): + Path to the video file. + sample_indices_fn (`Callable`, *optional*): + A callable function that will return indices at which the video should be sampled. If the video has to be loaded using + by a different sampling technique than provided by `num_frames` or `fps` arguments, one should provide their own `sample_indices_fn`. + If not provided, simple uniform sampling with fps is performed. + Example: + def sample_indices_fn(metadata, **kwargs): + return np.linspace(0, metadata.total_num_frames - 1, num_frames, dtype=int) + + Returns: + Tuple[`np.array`, `VideoMetadata`]: A tuple containing: + - Numpy array of frames in RGB (shape: [num_frames, height, width, 3]). + - `VideoMetadata` object. + """ + # Lazy import av + requires_backends(read_video_pyav, ["av"]) + import av + + container = av.open(video_path) + total_num_frames = container.streams.video[0].frames + video_fps = container.streams.video[0].average_rate # should we better use `av_guess_frame_rate`? + duration = total_num_frames / video_fps if video_fps else 0 + metadata = VideoMetadata( + total_num_frames=int(total_num_frames), fps=float(video_fps), duration=float(duration), video_backend="pyav" + ) + indices = sample_indices_fn(metadata=metadata, **kwargs) + + frames = [] + container.seek(0) + end_index = indices[-1] + for i, frame in enumerate(container.decode(video=0)): + if i > end_index: + break + if i >= 0 and i in indices: + frames.append(frame) + + video = np.stack([x.to_ndarray(format="rgb24") for x in frames]) + metadata.frames_indices = indices + return video, metadata + + +def read_video_torchvision( + video_path: str, + sample_indices_fn: Callable, + **kwargs, +): + """ + Decode the video with torchvision decoder. + + Args: + video_path (`str`): + Path to the video file. + sample_indices_fn (`Callable`, *optional*): + A callable function that will return indices at which the video should be sampled. If the video has to be loaded using + by a different sampling technique than provided by `num_frames` or `fps` arguments, one should provide their own `sample_indices_fn`. + If not provided, simple uniform sampling with fps is performed. + Example: + def sample_indices_fn(metadata, **kwargs): + return np.linspace(0, metadata.total_num_frames - 1, num_frames, dtype=int) + + Returns: + Tuple[`np.array`, `VideoMetadata`]: A tuple containing: + - Numpy array of frames in RGB (shape: [num_frames, height, width, 3]). + - `VideoMetadata` object. + """ + video, _, info = torchvision_io.read_video( + video_path, + start_pts=0.0, + end_pts=None, + pts_unit="sec", + output_format="THWC", + ) + video_fps = info["video_fps"] + total_num_frames = video.size(0) + duration = total_num_frames / video_fps if video_fps else 0 + metadata = VideoMetadata( + total_num_frames=int(total_num_frames), + fps=float(video_fps), + duration=float(duration), + video_backend="torchvision", + ) + + indices = sample_indices_fn(metadata=metadata, **kwargs) + + video = video[indices].contiguous().numpy() + metadata.frames_indices = indices + return video, metadata + + +VIDEO_DECODERS = { + "decord": read_video_decord, + "opencv": read_video_opencv, + "pyav": read_video_pyav, + "torchvision": read_video_torchvision, +} + + +def load_video( + video: Union[str, "VideoInput"], + num_frames: Optional[int] = None, + fps: Optional[int] = None, + backend: str = "opencv", + sample_indices_fn: Optional[Callable] = None, + **kwargs, +) -> np.array: + """ + Loads `video` to a numpy array. + + Args: + video (`str` or `VideoInput`): + The video to convert to the numpy array format. Can be a link to video or local path. + num_frames (`int`, *optional*): + Number of frames to sample uniformly. If not passed, the whole video is loaded. + fps (`int`, *optional*): + Number of frames to sample per second. Should be passed only when `num_frames=None`. + If not specified and `num_frames==None`, all frames are sampled. + backend (`str`, *optional*, defaults to `"opencv"`): + The backend to use when loading the video. Can be any of ["decord", "pyav", "opencv", "torchvision"]. Defaults to "opencv". + sample_indices_fn (`Callable`, *optional*): + A callable function that will return indices at which the video should be sampled. If the video has to be loaded using + by a different sampling technique than provided by `num_frames` or `fps` arguments, one should provide their own `sample_indices_fn`. + If not provided, simple uniformt sampling with fps is performed, otherwise `sample_indices_fn` has priority over other args. + The function expects at input the all args along with all kwargs passed to `load_video` and should output valid + indices at which the video should be sampled. For example: + + Example: + def sample_indices_fn(metadata, **kwargs): + return np.linspace(0, metadata.total_num_frames - 1, num_frames, dtype=int) + + Returns: + Tuple[`np.array`, Dict]: A tuple containing: + - Numpy array of frames in RGB (shape: [num_frames, height, width, 3]). + - Metadata dictionary. + """ + + # If `sample_indices_fn` is given, we can accept any args as those might be needed by custom `sample_indices_fn` + if fps is not None and num_frames is not None and sample_indices_fn is None: + raise ValueError( + "`num_frames`, `fps`, and `sample_indices_fn` are mutually exclusive arguments, please use only one!" + ) + + # If user didn't pass a sampling function, create one on the fly with default logic + if sample_indices_fn is None: + + def sample_indices_fn_func(metadata, **fn_kwargs): + return default_sample_indices_fn(metadata, num_frames=num_frames, fps=fps, **fn_kwargs) + + sample_indices_fn = sample_indices_fn_func + + if video.startswith("https://www.youtube.com") or video.startswith("http://www.youtube.com"): + if not is_yt_dlp_available(): + raise ImportError("To load a video from YouTube url you have to install `yt_dlp` first.") + # Lazy import from yt_dlp + requires_backends(load_video, ["yt_dlp"]) + from yt_dlp import YoutubeDL + + buffer = BytesIO() + with redirect_stdout(buffer), YoutubeDL() as f: + f.download([video]) + bytes_obj = buffer.getvalue() + file_obj = BytesIO(bytes_obj) + elif video.startswith("http://") or video.startswith("https://"): + file_obj = BytesIO(requests.get(video).content) + elif os.path.isfile(video): + file_obj = video + elif is_valid_image(video) or (isinstance(video, (list, tuple)) and is_valid_image(video[0])): + file_obj = None + else: + raise TypeError("Incorrect format used for video. Should be an url linking to an video or a local path.") + + # can also load with decord, but not cv2/torchvision + # both will fail in case of url links + video_is_url = video.startswith("http://") or video.startswith("https://") + if video_is_url and backend in ["opencv", "torchvision"]: + raise ValueError( + "If you are trying to load a video from URL, you can decode the video only with `pyav` or `decord` as backend" + ) + + if file_obj is None: + return video + + if ( + (not is_decord_available() and backend == "decord") + or (not is_av_available() and backend == "pyav") + or (not is_cv2_available() and backend == "opencv") + or (not is_torchvision_available() and backend == "torchvision") + ): + raise ImportError( + f"You chose backend={backend} for loading the video but the required library is not found in your environment " + f"Make sure to install {backend} before loading the video." + ) + + video_decoder = VIDEO_DECODERS[backend] + video, metadata = video_decoder(file_obj, sample_indices_fn, **kwargs) + return video, metadata + + +def load_images( + images: Union[list, tuple, str, "PIL.Image.Image"], timeout: Optional[float] = None +) -> Union["PIL.Image.Image", list["PIL.Image.Image"], list[list["PIL.Image.Image"]]]: + """Loads images, handling different levels of nesting. + + Args: + images: A single image, a list of images, or a list of lists of images to load. + timeout: Timeout for loading images. + + Returns: + A single image, a list of images, a list of lists of images. + """ + if isinstance(images, (list, tuple)): + if len(images) and isinstance(images[0], (list, tuple)): + return [[load_image(image, timeout=timeout) for image in image_group] for image_group in images] + else: + return [load_image(image, timeout=timeout) for image in images] + else: + return load_image(images, timeout=timeout) + + +def validate_preprocess_arguments( + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, list[float]]] = None, + image_std: Optional[Union[float, list[float]]] = None, + do_pad: Optional[bool] = None, + size_divisibility: Optional[int] = None, + do_center_crop: Optional[bool] = None, + crop_size: Optional[dict[str, int]] = None, + do_resize: Optional[bool] = None, + size: Optional[dict[str, int]] = None, + resample: Optional["PILImageResampling"] = None, +): + """ + Checks validity of typically used arguments in an `ImageProcessor` `preprocess` method. + Raises `ValueError` if arguments incompatibility is caught. + Many incompatibilities are model-specific. `do_pad` sometimes needs `size_divisor`, + sometimes `size_divisibility`, and sometimes `size`. New models and processors added should follow + existing arguments when possible. + + """ + if do_rescale and rescale_factor is None: + raise ValueError("`rescale_factor` must be specified if `do_rescale` is `True`.") + + if do_pad and size_divisibility is None: + # Here, size_divisor might be passed as the value of size + raise ValueError( + "Depending on the model, `size_divisibility`, `size_divisor`, `pad_size` or `size` must be specified if `do_pad` is `True`." + ) + + if do_normalize and (image_mean is None or image_std is None): + raise ValueError("`image_mean` and `image_std` must both be specified if `do_normalize` is `True`.") + + if do_center_crop and crop_size is None: + raise ValueError("`crop_size` must be specified if `do_center_crop` is `True`.") + + if do_resize and (size is None or resample is None): + raise ValueError("`size` and `resample` must be specified if `do_resize` is `True`.") + + +# In the future we can add a TF implementation here when we have TF models. +class ImageFeatureExtractionMixin: + """ + Mixin that contain utilities for preparing image features. + """ + + def _ensure_format_supported(self, image): + if not isinstance(image, (PIL.Image.Image, np.ndarray)) and not is_torch_tensor(image): + raise ValueError( + f"Got type {type(image)} which is not supported, only `PIL.Image.Image`, `np.array` and " + "`torch.Tensor` are." + ) + + def to_pil_image(self, image, rescale=None): + """ + Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last axis if + needed. + + Args: + image (`PIL.Image.Image` or `numpy.ndarray` or `torch.Tensor`): + The image to convert to the PIL Image format. + rescale (`bool`, *optional*): + Whether or not to apply the scaling factor (to make pixel values integers between 0 and 255). Will + default to `True` if the image type is a floating type, `False` otherwise. + """ + self._ensure_format_supported(image) + + if is_torch_tensor(image): + image = image.numpy() + + if isinstance(image, np.ndarray): + if rescale is None: + # rescale default to the array being of floating type. + rescale = isinstance(image.flat[0], np.floating) + # If the channel as been moved to first dim, we put it back at the end. + if image.ndim == 3 and image.shape[0] in [1, 3]: + image = image.transpose(1, 2, 0) + if rescale: + image = image * 255 + image = image.astype(np.uint8) + return PIL.Image.fromarray(image) + return image + + def convert_rgb(self, image): + """ + Converts `PIL.Image.Image` to RGB format. + + Args: + image (`PIL.Image.Image`): + The image to convert. + """ + self._ensure_format_supported(image) + if not isinstance(image, PIL.Image.Image): + return image + + return image.convert("RGB") + + def rescale(self, image: np.ndarray, scale: Union[float, int]) -> np.ndarray: + """ + Rescale a numpy image by scale amount + """ + self._ensure_format_supported(image) + return image * scale + + def to_numpy_array(self, image, rescale=None, channel_first=True): + """ + Converts `image` to a numpy array. Optionally rescales it and puts the channel dimension as the first + dimension. + + Args: + image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`): + The image to convert to a NumPy array. + rescale (`bool`, *optional*): + Whether or not to apply the scaling factor (to make pixel values floats between 0. and 1.). Will + default to `True` if the image is a PIL Image or an array/tensor of integers, `False` otherwise. + channel_first (`bool`, *optional*, defaults to `True`): + Whether or not to permute the dimensions of the image to put the channel dimension first. + """ + self._ensure_format_supported(image) + + if isinstance(image, PIL.Image.Image): + image = np.array(image) + + if is_torch_tensor(image): + image = image.numpy() + + rescale = isinstance(image.flat[0], np.integer) if rescale is None else rescale + + if rescale: + image = self.rescale(image.astype(np.float32), 1 / 255.0) + + if channel_first and image.ndim == 3: + image = image.transpose(2, 0, 1) + + return image + + def expand_dims(self, image): + """ + Expands 2-dimensional `image` to 3 dimensions. + + Args: + image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`): + The image to expand. + """ + self._ensure_format_supported(image) + + # Do nothing if PIL image + if isinstance(image, PIL.Image.Image): + return image + + if is_torch_tensor(image): + image = image.unsqueeze(0) + else: + image = np.expand_dims(image, axis=0) + return image + + def normalize(self, image, mean, std, rescale=False): + """ + Normalizes `image` with `mean` and `std`. Note that this will trigger a conversion of `image` to a NumPy array + if it's a PIL Image. + + Args: + image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`): + The image to normalize. + mean (`List[float]` or `np.ndarray` or `torch.Tensor`): + The mean (per channel) to use for normalization. + std (`List[float]` or `np.ndarray` or `torch.Tensor`): + The standard deviation (per channel) to use for normalization. + rescale (`bool`, *optional*, defaults to `False`): + Whether or not to rescale the image to be between 0 and 1. If a PIL image is provided, scaling will + happen automatically. + """ + self._ensure_format_supported(image) + + if isinstance(image, PIL.Image.Image): + image = self.to_numpy_array(image, rescale=True) + # If the input image is a PIL image, it automatically gets rescaled. If it's another + # type it may need rescaling. + elif rescale: + if isinstance(image, np.ndarray): + image = self.rescale(image.astype(np.float32), 1 / 255.0) + elif is_torch_tensor(image): + image = self.rescale(image.float(), 1 / 255.0) + + if isinstance(image, np.ndarray): + if not isinstance(mean, np.ndarray): + mean = np.array(mean).astype(image.dtype) + if not isinstance(std, np.ndarray): + std = np.array(std).astype(image.dtype) + elif is_torch_tensor(image): + import torch + + if not isinstance(mean, torch.Tensor): + if isinstance(mean, np.ndarray): + mean = torch.from_numpy(mean) + else: + mean = torch.tensor(mean) + if not isinstance(std, torch.Tensor): + if isinstance(std, np.ndarray): + std = torch.from_numpy(std) + else: + std = torch.tensor(std) + + if image.ndim == 3 and image.shape[0] in [1, 3]: + return (image - mean[:, None, None]) / std[:, None, None] + else: + return (image - mean) / std + + def resize(self, image, size, resample=None, default_to_square=True, max_size=None): + """ + Resizes `image`. Enforces conversion of input to PIL.Image. + + Args: + image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`): + The image to resize. + size (`int` or `Tuple[int, int]`): + The size to use for resizing the image. If `size` is a sequence like (h, w), output size will be + matched to this. + + If `size` is an int and `default_to_square` is `True`, then image will be resized to (size, size). If + `size` is an int and `default_to_square` is `False`, then smaller edge of the image will be matched to + this number. i.e, if height > width, then image will be rescaled to (size * height / width, size). + resample (`int`, *optional*, defaults to `PILImageResampling.BILINEAR`): + The filter to user for resampling. + default_to_square (`bool`, *optional*, defaults to `True`): + How to convert `size` when it is a single int. If set to `True`, the `size` will be converted to a + square (`size`,`size`). If set to `False`, will replicate + [`torchvision.transforms.Resize`](https://pytorch.org/vision/stable/transforms.html#torchvision.transforms.Resize) + with support for resizing only the smallest edge and providing an optional `max_size`. + max_size (`int`, *optional*, defaults to `None`): + The maximum allowed for the longer edge of the resized image: if the longer edge of the image is + greater than `max_size` after being resized according to `size`, then the image is resized again so + that the longer edge is equal to `max_size`. As a result, `size` might be overruled, i.e the smaller + edge may be shorter than `size`. Only used if `default_to_square` is `False`. + + Returns: + image: A resized `PIL.Image.Image`. + """ + resample = resample if resample is not None else PILImageResampling.BILINEAR + + self._ensure_format_supported(image) + + if not isinstance(image, PIL.Image.Image): + image = self.to_pil_image(image) + + if isinstance(size, list): + size = tuple(size) + + if isinstance(size, int) or len(size) == 1: + if default_to_square: + size = (size, size) if isinstance(size, int) else (size[0], size[0]) + else: + width, height = image.size + # specified size only for the smallest edge + short, long = (width, height) if width <= height else (height, width) + requested_new_short = size if isinstance(size, int) else size[0] + + if short == requested_new_short: + return image + + new_short, new_long = requested_new_short, int(requested_new_short * long / short) + + if max_size is not None: + if max_size <= requested_new_short: + raise ValueError( + f"max_size = {max_size} must be strictly greater than the requested " + f"size for the smaller edge size = {size}" + ) + if new_long > max_size: + new_short, new_long = int(max_size * new_short / new_long), max_size + + size = (new_short, new_long) if width <= height else (new_long, new_short) + + return image.resize(size, resample=resample) + + def center_crop(self, image, size): + """ + Crops `image` to the given size using a center crop. Note that if the image is too small to be cropped to the + size given, it will be padded (so the returned result has the size asked). + + Args: + image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor` of shape (n_channels, height, width) or (height, width, n_channels)): + The image to resize. + size (`int` or `Tuple[int, int]`): + The size to which crop the image. + + Returns: + new_image: A center cropped `PIL.Image.Image` or `np.ndarray` or `torch.Tensor` of shape: (n_channels, + height, width). + """ + self._ensure_format_supported(image) + + if not isinstance(size, tuple): + size = (size, size) + + # PIL Image.size is (width, height) but NumPy array and torch Tensors have (height, width) + if is_torch_tensor(image) or isinstance(image, np.ndarray): + if image.ndim == 2: + image = self.expand_dims(image) + image_shape = image.shape[1:] if image.shape[0] in [1, 3] else image.shape[:2] + else: + image_shape = (image.size[1], image.size[0]) + + top = (image_shape[0] - size[0]) // 2 + bottom = top + size[0] # In case size is odd, (image_shape[0] + size[0]) // 2 won't give the proper result. + left = (image_shape[1] - size[1]) // 2 + right = left + size[1] # In case size is odd, (image_shape[1] + size[1]) // 2 won't give the proper result. + + # For PIL Images we have a method to crop directly. + if isinstance(image, PIL.Image.Image): + return image.crop((left, top, right, bottom)) + + # Check if image is in (n_channels, height, width) or (height, width, n_channels) format + channel_first = True if image.shape[0] in [1, 3] else False + + # Transpose (height, width, n_channels) format images + if not channel_first: + if isinstance(image, np.ndarray): + image = image.transpose(2, 0, 1) + if is_torch_tensor(image): + image = image.permute(2, 0, 1) + + # Check if cropped area is within image boundaries + if top >= 0 and bottom <= image_shape[0] and left >= 0 and right <= image_shape[1]: + return image[..., top:bottom, left:right] + + # Otherwise, we may need to pad if the image is too small. Oh joy... + new_shape = image.shape[:-2] + (max(size[0], image_shape[0]), max(size[1], image_shape[1])) + if isinstance(image, np.ndarray): + new_image = np.zeros_like(image, shape=new_shape) + elif is_torch_tensor(image): + new_image = image.new_zeros(new_shape) + + top_pad = (new_shape[-2] - image_shape[0]) // 2 + bottom_pad = top_pad + image_shape[0] + left_pad = (new_shape[-1] - image_shape[1]) // 2 + right_pad = left_pad + image_shape[1] + new_image[..., top_pad:bottom_pad, left_pad:right_pad] = image + + top += top_pad + bottom += top_pad + left += left_pad + right += left_pad + + new_image = new_image[ + ..., max(0, top) : min(new_image.shape[-2], bottom), max(0, left) : min(new_image.shape[-1], right) + ] + + return new_image + + def flip_channel_order(self, image): + """ + Flips the channel order of `image` from RGB to BGR, or vice versa. Note that this will trigger a conversion of + `image` to a NumPy array if it's a PIL Image. + + Args: + image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`): + The image whose color channels to flip. If `np.ndarray` or `torch.Tensor`, the channel dimension should + be first. + """ + self._ensure_format_supported(image) + + if isinstance(image, PIL.Image.Image): + image = self.to_numpy_array(image) + + return image[::-1, :, :] + + def rotate(self, image, angle, resample=None, expand=0, center=None, translate=None, fillcolor=None): + """ + Returns a rotated copy of `image`. This method returns a copy of `image`, rotated the given number of degrees + counter clockwise around its centre. + + Args: + image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`): + The image to rotate. If `np.ndarray` or `torch.Tensor`, will be converted to `PIL.Image.Image` before + rotating. + + Returns: + image: A rotated `PIL.Image.Image`. + """ + resample = resample if resample is not None else PIL.Image.NEAREST + + self._ensure_format_supported(image) + + if not isinstance(image, PIL.Image.Image): + image = self.to_pil_image(image) + + return image.rotate( + angle, resample=resample, expand=expand, center=center, translate=translate, fillcolor=fillcolor + ) + + +def validate_annotations( + annotation_format: AnnotationFormat, + supported_annotation_formats: tuple[AnnotationFormat, ...], + annotations: list[dict], +) -> None: + if annotation_format not in supported_annotation_formats: + raise ValueError(f"Unsupported annotation format: {format} must be one of {supported_annotation_formats}") + + if annotation_format is AnnotationFormat.COCO_DETECTION: + if not valid_coco_detection_annotations(annotations): + raise ValueError( + "Invalid COCO detection annotations. Annotations must a dict (single image) or list of dicts " + "(batch of images) with the following keys: `image_id` and `annotations`, with the latter " + "being a list of annotations in the COCO format." + ) + + if annotation_format is AnnotationFormat.COCO_PANOPTIC: + if not valid_coco_panoptic_annotations(annotations): + raise ValueError( + "Invalid COCO panoptic annotations. Annotations must a dict (single image) or list of dicts " + "(batch of images) with the following keys: `image_id`, `file_name` and `segments_info`, with " + "the latter being a list of annotations in the COCO format." + ) + + +def validate_kwargs(valid_processor_keys: list[str], captured_kwargs: list[str]): + unused_keys = set(captured_kwargs).difference(set(valid_processor_keys)) + if unused_keys: + unused_key_str = ", ".join(unused_keys) + # TODO raise a warning here instead of simply logging? + logger.warning(f"Unused or unrecognized kwargs: {unused_key_str}.") + + +@dataclass(frozen=True) +class SizeDict: + """ + Hashable dictionary to store image size information. + """ + + height: Optional[int] = None + width: Optional[int] = None + longest_edge: Optional[int] = None + shortest_edge: Optional[int] = None + max_height: Optional[int] = None + max_width: Optional[int] = None + + def __getitem__(self, key): + if hasattr(self, key): + return getattr(self, key) + raise KeyError(f"Key {key} not found in SizeDict.") diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/import_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/import_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..61fb91e5a844a04fcb7f2eab9b259754964ffde6 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/import_utils.py @@ -0,0 +1,2417 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Import utilities: Utilities related to imports and our lazy inits. +""" + +import importlib.machinery +import importlib.metadata +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import warnings +from collections import OrderedDict +from functools import lru_cache +from itertools import chain +from types import ModuleType +from typing import Any, Dict, FrozenSet, Optional, Set, Tuple, Union + +from packaging import version + +from . import logging + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# TODO: This doesn't work for all packages (`bs4`, `faiss`, etc.) Talk to Sylvain to see how to do with it better. +def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]: + # Check if the package spec exists and grab its version to avoid importing a local directory + package_exists = importlib.util.find_spec(pkg_name) is not None + package_version = "N/A" + if package_exists: + try: + # TODO: Once python 3.9 support is dropped, `importlib.metadata.packages_distributions()` + # should be used here to map from package name to distribution names + # e.g. PIL -> Pillow, Pillow-SIMD; quark -> amd-quark; onnxruntime -> onnxruntime-gpu. + # `importlib.metadata.packages_distributions()` is not available in Python 3.9. + + # Primary method to get the package version + package_version = importlib.metadata.version(pkg_name) + except importlib.metadata.PackageNotFoundError: + # Fallback method: Only for "torch" and versions containing "dev" + if pkg_name == "torch": + try: + package = importlib.import_module(pkg_name) + temp_version = getattr(package, "__version__", "N/A") + # Check if the version contains "dev" + if "dev" in temp_version: + package_version = temp_version + package_exists = True + else: + package_exists = False + except ImportError: + # If the package can't be imported, it's not available + package_exists = False + elif pkg_name == "quark": + # TODO: remove once `importlib.metadata.packages_distributions()` is supported. + try: + package_version = importlib.metadata.version("amd-quark") + except Exception: + package_exists = False + else: + # For packages other than "torch", don't attempt the fallback and set as not available + package_exists = False + logger.debug(f"Detected {pkg_name} version: {package_version}") + if return_version: + return package_exists, package_version + else: + return package_exists + + +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) + +USE_TF = os.environ.get("USE_TF", "AUTO").upper() +USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper() +USE_JAX = os.environ.get("USE_FLAX", "AUTO").upper() + +# Try to run a native pytorch job in an environment with TorchXLA installed by setting this value to 0. +USE_TORCH_XLA = os.environ.get("USE_TORCH_XLA", "1").upper() + +FORCE_TF_AVAILABLE = os.environ.get("FORCE_TF_AVAILABLE", "AUTO").upper() + +# `transformers` requires `torch>=1.11` but this variable is exposed publicly, and we can't simply remove it. +# This is the version of torch required to run torch.fx features and torch.onnx with dictionary inputs. +TORCH_FX_REQUIRED_VERSION = version.parse("1.10") + +ACCELERATE_MIN_VERSION = "0.26.0" +SCHEDULEFREE_MIN_VERSION = "1.2.6" +FSDP_MIN_VERSION = "1.12.0" +GGUF_MIN_VERSION = "0.10.0" +XLA_FSDPV2_MIN_VERSION = "2.2.0" +HQQ_MIN_VERSION = "0.2.1" +VPTQ_MIN_VERSION = "0.0.4" +TORCHAO_MIN_VERSION = "0.4.0" + + +_accelerate_available, _accelerate_version = _is_package_available("accelerate", return_version=True) +_apex_available = _is_package_available("apex") +_apollo_torch_available = _is_package_available("apollo_torch") +_aqlm_available = _is_package_available("aqlm") +_vptq_available, _vptq_version = _is_package_available("vptq", return_version=True) +_av_available = importlib.util.find_spec("av") is not None +_decord_available = importlib.util.find_spec("decord") is not None +_bitsandbytes_available = _is_package_available("bitsandbytes") +_eetq_available = _is_package_available("eetq") +_fbgemm_gpu_available = _is_package_available("fbgemm_gpu") +_galore_torch_available = _is_package_available("galore_torch") +_lomo_available = _is_package_available("lomo_optim") +_grokadamw_available = _is_package_available("grokadamw") +_schedulefree_available, _schedulefree_version = _is_package_available("schedulefree", return_version=True) +# `importlib.metadata.version` doesn't work with `bs4` but `beautifulsoup4`. For `importlib.util.find_spec`, reversed. +_bs4_available = importlib.util.find_spec("bs4") is not None +_coloredlogs_available = _is_package_available("coloredlogs") +# `importlib.metadata.util` doesn't work with `opencv-python-headless`. +_cv2_available = importlib.util.find_spec("cv2") is not None +_yt_dlp_available = importlib.util.find_spec("yt_dlp") is not None +_datasets_available = _is_package_available("datasets") +_detectron2_available = _is_package_available("detectron2") +# We need to check both `faiss` and `faiss-cpu`. +_faiss_available = importlib.util.find_spec("faiss") is not None +try: + _faiss_version = importlib.metadata.version("faiss") + logger.debug(f"Successfully imported faiss version {_faiss_version}") +except importlib.metadata.PackageNotFoundError: + try: + _faiss_version = importlib.metadata.version("faiss-cpu") + logger.debug(f"Successfully imported faiss version {_faiss_version}") + except importlib.metadata.PackageNotFoundError: + _faiss_available = False +_ftfy_available = _is_package_available("ftfy") +_g2p_en_available = _is_package_available("g2p_en") +_hadamard_available = _is_package_available("fast_hadamard_transform") +_ipex_available, _ipex_version = _is_package_available("intel_extension_for_pytorch", return_version=True) +_jieba_available = _is_package_available("jieba") +_jinja_available = _is_package_available("jinja2") +_kenlm_available = _is_package_available("kenlm") +_keras_nlp_available = _is_package_available("keras_nlp") +_levenshtein_available = _is_package_available("Levenshtein") +_librosa_available = _is_package_available("librosa") +_natten_available = _is_package_available("natten") +_nltk_available = _is_package_available("nltk") +_onnx_available = _is_package_available("onnx") +_openai_available = _is_package_available("openai") +_optimum_available = _is_package_available("optimum") +_auto_gptq_available = _is_package_available("auto_gptq") +_gptqmodel_available = _is_package_available("gptqmodel") +# `importlib.metadata.version` doesn't work with `awq` +_auto_awq_available = importlib.util.find_spec("awq") is not None +_quark_available = _is_package_available("quark") +_is_optimum_quanto_available = False +try: + importlib.metadata.version("optimum_quanto") + _is_optimum_quanto_available = True +except importlib.metadata.PackageNotFoundError: + _is_optimum_quanto_available = False +# For compressed_tensors, only check spec to allow compressed_tensors-nightly package +_compressed_tensors_available = importlib.util.find_spec("compressed_tensors") is not None +_pandas_available = _is_package_available("pandas") +_peft_available = _is_package_available("peft") +_phonemizer_available = _is_package_available("phonemizer") +_uroman_available = _is_package_available("uroman") +_psutil_available = _is_package_available("psutil") +_py3nvml_available = _is_package_available("py3nvml") +_pyctcdecode_available = _is_package_available("pyctcdecode") +_pygments_available = _is_package_available("pygments") +_pytesseract_available = _is_package_available("pytesseract") +_pytest_available = _is_package_available("pytest") +_pytorch_quantization_available = _is_package_available("pytorch_quantization") +_rjieba_available = _is_package_available("rjieba") +_sacremoses_available = _is_package_available("sacremoses") +_safetensors_available = _is_package_available("safetensors") +_scipy_available = _is_package_available("scipy") +_sentencepiece_available = _is_package_available("sentencepiece") +_is_seqio_available = _is_package_available("seqio") +_is_gguf_available, _gguf_version = _is_package_available("gguf", return_version=True) +_sklearn_available = importlib.util.find_spec("sklearn") is not None +if _sklearn_available: + try: + importlib.metadata.version("scikit-learn") + except importlib.metadata.PackageNotFoundError: + _sklearn_available = False +_smdistributed_available = importlib.util.find_spec("smdistributed") is not None +_soundfile_available = _is_package_available("soundfile") +_spacy_available = _is_package_available("spacy") +_sudachipy_available, _sudachipy_version = _is_package_available("sudachipy", return_version=True) +_tensorflow_probability_available = _is_package_available("tensorflow_probability") +_tensorflow_text_available = _is_package_available("tensorflow_text") +_tf2onnx_available = _is_package_available("tf2onnx") +_timm_available = _is_package_available("timm") +_tokenizers_available = _is_package_available("tokenizers") +_torchaudio_available = _is_package_available("torchaudio") +_torchao_available, _torchao_version = _is_package_available("torchao", return_version=True) +_torchdistx_available = _is_package_available("torchdistx") +_torchvision_available, _torchvision_version = _is_package_available("torchvision", return_version=True) +_mlx_available = _is_package_available("mlx") +_num2words_available = _is_package_available("num2words") +_hqq_available, _hqq_version = _is_package_available("hqq", return_version=True) +_tiktoken_available = _is_package_available("tiktoken") +_blobfile_available = _is_package_available("blobfile") +_liger_kernel_available = _is_package_available("liger_kernel") +_triton_available = _is_package_available("triton") +_spqr_available = _is_package_available("spqr_quant") +_rich_available = _is_package_available("rich") + +_torch_version = "N/A" +_torch_available = False +if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES: + _torch_available, _torch_version = _is_package_available("torch", return_version=True) +else: + logger.info("Disabling PyTorch because USE_TF is set") + _torch_available = False + + +_tf_version = "N/A" +_tf_available = False +if FORCE_TF_AVAILABLE in ENV_VARS_TRUE_VALUES: + _tf_available = True +else: + if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES: + # Note: _is_package_available("tensorflow") fails for tensorflow-cpu. Please test any changes to the line below + # with tensorflow-cpu to make sure it still works! + _tf_available = importlib.util.find_spec("tensorflow") is not None + if _tf_available: + candidates = ( + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "tf-nightly-rocm", + "intel-tensorflow", + "intel-tensorflow-avx512", + "tensorflow-rocm", + "tensorflow-macos", + "tensorflow-aarch64", + ) + _tf_version = None + # For the metadata, we have to look for both tensorflow and tensorflow-cpu + for pkg in candidates: + try: + _tf_version = importlib.metadata.version(pkg) + break + except importlib.metadata.PackageNotFoundError: + pass + _tf_available = _tf_version is not None + if _tf_available: + if version.parse(_tf_version) < version.parse("2"): + logger.info( + f"TensorFlow found but with version {_tf_version}. Transformers requires version 2 minimum." + ) + _tf_available = False + else: + logger.info("Disabling Tensorflow because USE_TORCH is set") + + +_essentia_available = importlib.util.find_spec("essentia") is not None +try: + _essentia_version = importlib.metadata.version("essentia") + logger.debug(f"Successfully imported essentia version {_essentia_version}") +except importlib.metadata.PackageNotFoundError: + _essentia_version = False + + +_pretty_midi_available = importlib.util.find_spec("pretty_midi") is not None +try: + _pretty_midi_version = importlib.metadata.version("pretty_midi") + logger.debug(f"Successfully imported pretty_midi version {_pretty_midi_version}") +except importlib.metadata.PackageNotFoundError: + _pretty_midi_available = False + + +ccl_version = "N/A" +_is_ccl_available = ( + importlib.util.find_spec("torch_ccl") is not None + or importlib.util.find_spec("oneccl_bindings_for_pytorch") is not None +) +try: + ccl_version = importlib.metadata.version("oneccl_bind_pt") + logger.debug(f"Detected oneccl_bind_pt version {ccl_version}") +except importlib.metadata.PackageNotFoundError: + _is_ccl_available = False + + +_flax_available = False +if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES: + _flax_available, _flax_version = _is_package_available("flax", return_version=True) + if _flax_available: + _jax_available, _jax_version = _is_package_available("jax", return_version=True) + if _jax_available: + logger.info(f"JAX version {_jax_version}, Flax version {_flax_version} available.") + else: + _flax_available = _jax_available = False + _jax_version = _flax_version = "N/A" + + +_torch_fx_available = False +if _torch_available: + torch_version = version.parse(_torch_version) + _torch_fx_available = (torch_version.major, torch_version.minor) >= ( + TORCH_FX_REQUIRED_VERSION.major, + TORCH_FX_REQUIRED_VERSION.minor, + ) + + +_torch_xla_available = False +if USE_TORCH_XLA in ENV_VARS_TRUE_VALUES: + _torch_xla_available, _torch_xla_version = _is_package_available("torch_xla", return_version=True) + if _torch_xla_available: + logger.info(f"Torch XLA version {_torch_xla_version} available.") + + +def is_kenlm_available(): + return _kenlm_available + + +def is_cv2_available(): + return _cv2_available + + +def is_yt_dlp_available(): + return _yt_dlp_available + + +def is_torch_available(): + return _torch_available + + +def is_accelerate_available(min_version: str = ACCELERATE_MIN_VERSION): + return _accelerate_available and version.parse(_accelerate_version) >= version.parse(min_version) + + +def is_torch_deterministic(): + """ + Check whether pytorch uses deterministic algorithms by looking if torch.set_deterministic_debug_mode() is set to 1 or 2" + """ + import torch + + if torch.get_deterministic_debug_mode() == 0: + return False + else: + return True + + +def is_hadamard_available(): + return _hadamard_available + + +def is_hqq_available(min_version: str = HQQ_MIN_VERSION): + return _hqq_available and version.parse(_hqq_version) >= version.parse(min_version) + + +def is_pygments_available(): + return _pygments_available + + +def get_torch_version(): + return _torch_version + + +def is_torch_sdpa_available(): + if not is_torch_available(): + return False + elif _torch_version == "N/A": + return False + + # NOTE: We require torch>=2.1 (and not torch>=2.0) to use SDPA in Transformers for two reasons: + # - Allow the global use of the `scale` argument introduced in https://github.com/pytorch/pytorch/pull/95259 + # - Memory-efficient attention supports arbitrary attention_mask: https://github.com/pytorch/pytorch/pull/104310 + # NOTE: MLU is OK with non-contiguous inputs. + if is_torch_mlu_available(): + return version.parse(_torch_version) >= version.parse("2.1.0") + # NOTE: NPU can use SDPA in Transformers with torch>=2.1.0. + if is_torch_npu_available(): + return version.parse(_torch_version) >= version.parse("2.1.0") + # NOTE: We require torch>=2.1.1 to avoid a numerical issue in SDPA with non-contiguous inputs: https://github.com/pytorch/pytorch/issues/112577 + return version.parse(_torch_version) >= version.parse("2.1.1") + + +def is_torch_flex_attn_available(): + if not is_torch_available(): + return False + elif _torch_version == "N/A": + return False + + # TODO check if some bugs cause push backs on the exact version + # NOTE: We require torch>=2.5.0 as it is the first release + return version.parse(_torch_version) >= version.parse("2.5.0") + + +def is_torchvision_available(): + return _torchvision_available + + +def is_torchvision_v2_available(): + if not is_torchvision_available(): + return False + + # NOTE: We require torchvision>=0.15 as v2 transforms are available from this version: https://pytorch.org/vision/stable/transforms.html#v1-or-v2-which-one-should-i-use + return version.parse(_torchvision_version) >= version.parse("0.15") + + +def is_galore_torch_available(): + return _galore_torch_available + + +def is_apollo_torch_available(): + return _apollo_torch_available + + +def is_lomo_available(): + return _lomo_available + + +def is_grokadamw_available(): + return _grokadamw_available + + +def is_schedulefree_available(min_version: str = SCHEDULEFREE_MIN_VERSION): + return _schedulefree_available and version.parse(_schedulefree_version) >= version.parse(min_version) + + +def is_pyctcdecode_available(): + return _pyctcdecode_available + + +def is_librosa_available(): + return _librosa_available + + +def is_essentia_available(): + return _essentia_available + + +def is_pretty_midi_available(): + return _pretty_midi_available + + +def is_torch_cuda_available(): + if is_torch_available(): + import torch + + return torch.cuda.is_available() + else: + return False + + +def is_mamba_ssm_available(): + if is_torch_available(): + import torch + + if not torch.cuda.is_available(): + return False + else: + return _is_package_available("mamba_ssm") + return False + + +def is_mamba_2_ssm_available(): + if is_torch_available(): + import torch + + if not torch.cuda.is_available(): + return False + else: + if _is_package_available("mamba_ssm"): + import mamba_ssm + + if version.parse(mamba_ssm.__version__) >= version.parse("2.0.4"): + return True + return False + + +def is_causal_conv1d_available(): + if is_torch_available(): + import torch + + if not torch.cuda.is_available(): + return False + return _is_package_available("causal_conv1d") + return False + + +def is_mambapy_available(): + if is_torch_available(): + return _is_package_available("mambapy") + return False + + +def is_torch_mps_available(min_version: Optional[str] = None): + if is_torch_available(): + import torch + + if hasattr(torch.backends, "mps"): + backend_available = torch.backends.mps.is_available() and torch.backends.mps.is_built() + if min_version is not None: + flag = version.parse(_torch_version) >= version.parse(min_version) + backend_available = backend_available and flag + return backend_available + return False + + +def is_torch_bf16_gpu_available(): + if not is_torch_available(): + return False + + import torch + + return torch.cuda.is_available() and torch.cuda.is_bf16_supported() + + +def is_torch_bf16_cpu_available(): + if not is_torch_available(): + return False + + import torch + + try: + # multiple levels of AttributeError depending on the pytorch version so do them all in one check + _ = torch.cpu.amp.autocast + except AttributeError: + return False + + return True + + +def is_torch_bf16_available(): + # the original bf16 check was for gpu only, but later a cpu/bf16 combo has emerged so this util + # has become ambiguous and therefore deprecated + warnings.warn( + "The util is_torch_bf16_available is deprecated, please use is_torch_bf16_gpu_available " + "or is_torch_bf16_cpu_available instead according to whether it's used with cpu or gpu", + FutureWarning, + ) + return is_torch_bf16_gpu_available() + + +@lru_cache() +def is_torch_fp16_available_on_device(device): + if not is_torch_available(): + return False + + if is_torch_hpu_available(): + if is_habana_gaudi1(): + return False + else: + return True + + import torch + + try: + x = torch.zeros(2, 2, dtype=torch.float16, device=device) + _ = x @ x + + # At this moment, let's be strict of the check: check if `LayerNorm` is also supported on device, because many + # models use this layer. + batch, sentence_length, embedding_dim = 3, 4, 5 + embedding = torch.randn(batch, sentence_length, embedding_dim, dtype=torch.float16, device=device) + layer_norm = torch.nn.LayerNorm(embedding_dim, dtype=torch.float16, device=device) + _ = layer_norm(embedding) + + except: # noqa: E722 + # TODO: more precise exception matching, if possible. + # most backends should return `RuntimeError` however this is not guaranteed. + return False + + return True + + +@lru_cache() +def is_torch_bf16_available_on_device(device): + if not is_torch_available(): + return False + + import torch + + if device == "cuda": + return is_torch_bf16_gpu_available() + + if device == "hpu": + return True + + try: + x = torch.zeros(2, 2, dtype=torch.bfloat16, device=device) + _ = x @ x + except: # noqa: E722 + # TODO: more precise exception matching, if possible. + # most backends should return `RuntimeError` however this is not guaranteed. + return False + + return True + + +def is_torch_tf32_available(): + if not is_torch_available(): + return False + + import torch + + if not torch.cuda.is_available() or torch.version.cuda is None: + return False + if torch.cuda.get_device_properties(torch.cuda.current_device()).major < 8: + return False + if int(torch.version.cuda.split(".")[0]) < 11: + return False + if version.parse(version.parse(torch.__version__).base_version) < version.parse("1.7"): + return False + + return True + + +def is_torch_fx_available(): + return _torch_fx_available + + +def is_peft_available(): + return _peft_available + + +def is_bs4_available(): + return _bs4_available + + +def is_tf_available(): + return _tf_available + + +def is_coloredlogs_available(): + return _coloredlogs_available + + +def is_tf2onnx_available(): + return _tf2onnx_available + + +def is_onnx_available(): + return _onnx_available + + +def is_openai_available(): + return _openai_available + + +def is_flax_available(): + return _flax_available + + +def is_flute_available(): + try: + return importlib.util.find_spec("flute") is not None and importlib.metadata.version("flute-kernel") >= "0.4.1" + except importlib.metadata.PackageNotFoundError: + return False + + +def is_ftfy_available(): + return _ftfy_available + + +def is_g2p_en_available(): + return _g2p_en_available + + +@lru_cache +def is_torch_xla_available(check_is_tpu=False, check_is_gpu=False): + """ + Check if `torch_xla` is available. To train a native pytorch job in an environment with torch xla installed, set + the USE_TORCH_XLA to false. + """ + assert not (check_is_tpu and check_is_gpu), "The check_is_tpu and check_is_gpu cannot both be true." + + if not _torch_xla_available: + return False + + import torch_xla + + if check_is_gpu: + return torch_xla.runtime.device_type() in ["GPU", "CUDA"] + elif check_is_tpu: + return torch_xla.runtime.device_type() == "TPU" + + return True + + +@lru_cache() +def is_torch_neuroncore_available(check_device=True): + if importlib.util.find_spec("torch_neuronx") is not None: + return is_torch_xla_available() + return False + + +@lru_cache() +def is_torch_npu_available(check_device=False): + "Checks if `torch_npu` is installed and potentially if a NPU is in the environment" + if not _torch_available or importlib.util.find_spec("torch_npu") is None: + return False + + import torch + import torch_npu # noqa: F401 + + if check_device: + try: + # Will raise a RuntimeError if no NPU is found + _ = torch.npu.device_count() + return torch.npu.is_available() + except RuntimeError: + return False + return hasattr(torch, "npu") and torch.npu.is_available() + + +@lru_cache() +def is_torch_mlu_available(check_device=False): + """ + Checks if `mlu` is available via an `cndev-based` check which won't trigger the drivers and leave mlu + uninitialized. + """ + if not _torch_available or importlib.util.find_spec("torch_mlu") is None: + return False + + import torch + import torch_mlu # noqa: F401 + + pytorch_cndev_based_mlu_check_previous_value = os.environ.get("PYTORCH_CNDEV_BASED_MLU_CHECK") + try: + os.environ["PYTORCH_CNDEV_BASED_MLU_CHECK"] = str(1) + available = torch.mlu.is_available() + finally: + if pytorch_cndev_based_mlu_check_previous_value: + os.environ["PYTORCH_CNDEV_BASED_MLU_CHECK"] = pytorch_cndev_based_mlu_check_previous_value + else: + os.environ.pop("PYTORCH_CNDEV_BASED_MLU_CHECK", None) + + return available + + +@lru_cache() +def is_torch_musa_available(check_device=False): + "Checks if `torch_musa` is installed and potentially if a MUSA is in the environment" + if not _torch_available or importlib.util.find_spec("torch_musa") is None: + return False + + import torch + import torch_musa # noqa: F401 + + torch_musa_min_version = "0.33.0" + if _accelerate_available and version.parse(_accelerate_version) < version.parse(torch_musa_min_version): + return False + + if check_device: + try: + # Will raise a RuntimeError if no MUSA is found + _ = torch.musa.device_count() + return torch.musa.is_available() + except RuntimeError: + return False + return hasattr(torch, "musa") and torch.musa.is_available() + + +@lru_cache +def is_torch_hpu_available(): + "Checks if `torch.hpu` is available and potentially if a HPU is in the environment" + if ( + not _torch_available + or importlib.util.find_spec("habana_frameworks") is None + or importlib.util.find_spec("habana_frameworks.torch") is None + ): + return False + + torch_hpu_min_version = "1.5.0" + if _accelerate_available and version.parse(_accelerate_version) < version.parse(torch_hpu_min_version): + return False + + import torch + + if os.environ.get("PT_HPU_LAZY_MODE", "1") == "1": + # import habana_frameworks.torch in case of lazy mode to patch torch with torch.hpu + import habana_frameworks.torch # noqa: F401 + + if not hasattr(torch, "hpu") or not torch.hpu.is_available(): + return False + + import habana_frameworks.torch.utils.experimental as htexp # noqa: F401 + + # IlyasMoutawwakil: We patch masked_fill_ for int64 tensors to avoid a bug on Gaudi1 + # synNodeCreateWithId failed for node: masked_fill_fwd_i64 with synStatus 26 [Generic failure] + # This can be removed once Gaudi1 support is discontinued but for now we need it to keep using + # dl1.24xlarge Gaudi1 instances on AWS for testing. + # check if the device is Gaudi1 (vs Gaudi2, Gaudi3). + if htexp._get_device_type() == htexp.synDeviceType.synDeviceGaudi: + original_masked_fill_ = torch.Tensor.masked_fill_ + + def patched_masked_fill_(self, mask, value): + if self.dtype == torch.int64: + logger.warning_once( + "In-place tensor.masked_fill_(mask, value) is not supported for int64 tensors on Gaudi1. " + "This operation will be performed out-of-place using tensor[mask] = value." + ) + self[mask] = value + else: + original_masked_fill_(self, mask, value) + + torch.Tensor.masked_fill_ = patched_masked_fill_ + + return True + + +@lru_cache +def is_habana_gaudi1(): + if not is_torch_hpu_available(): + return False + + import habana_frameworks.torch.utils.experimental as htexp # noqa: F401 + + # Check if the device is Gaudi1 (vs Gaudi2, Gaudi3) + return htexp._get_device_type() == htexp.synDeviceType.synDeviceGaudi + + +def is_torchdynamo_available(): + if not is_torch_available(): + return False + + return version.parse(_torch_version) >= version.parse("2.0.0") + + +def is_torch_compile_available(): + if not is_torch_available(): + return False + + import torch + + # We don't do any version check here to support nighlies marked as 1.14. Ultimately needs to check version against + # 2.0 but let's do it later. + return hasattr(torch, "compile") + + +def is_torchdynamo_compiling(): + if not is_torch_available(): + return False + + # Importing torch._dynamo causes issues with PyTorch profiler (https://github.com/pytorch/pytorch/issues/130622) + # hence rather relying on `torch.compiler.is_compiling()` when possible (torch>=2.3) + try: + import torch + + return torch.compiler.is_compiling() + except Exception: + try: + import torch._dynamo as dynamo # noqa: F401 + + return dynamo.is_compiling() + except Exception: + return False + + +def is_torchdynamo_exporting(): + if not is_torch_available(): + return False + + try: + import torch + + return torch.compiler.is_exporting() + except Exception: + try: + import torch._dynamo as dynamo # noqa: F401 + + return dynamo.is_exporting() + except Exception: + return False + + +def is_torch_tensorrt_fx_available(): + if importlib.util.find_spec("torch_tensorrt") is None: + return False + return importlib.util.find_spec("torch_tensorrt.fx") is not None + + +def is_datasets_available(): + return _datasets_available + + +def is_detectron2_available(): + return _detectron2_available + + +def is_rjieba_available(): + return _rjieba_available + + +def is_psutil_available(): + return _psutil_available + + +def is_py3nvml_available(): + return _py3nvml_available + + +def is_sacremoses_available(): + return _sacremoses_available + + +def is_apex_available(): + return _apex_available + + +def is_aqlm_available(): + return _aqlm_available + + +def is_vptq_available(min_version: str = VPTQ_MIN_VERSION): + return _vptq_available and version.parse(_vptq_version) >= version.parse(min_version) + + +def is_av_available(): + return _av_available + + +def is_decord_available(): + return _decord_available + + +def is_ninja_available(): + r""" + Code comes from *torch.utils.cpp_extension.is_ninja_available()*. Returns `True` if the + [ninja](https://ninja-build.org/) build system is available on the system, `False` otherwise. + """ + try: + subprocess.check_output("ninja --version".split()) + except Exception: + return False + else: + return True + + +def is_ipex_available(min_version: str = ""): + def get_major_and_minor_from_version(full_version): + return str(version.parse(full_version).major) + "." + str(version.parse(full_version).minor) + + if not is_torch_available() or not _ipex_available: + return False + + torch_major_and_minor = get_major_and_minor_from_version(_torch_version) + ipex_major_and_minor = get_major_and_minor_from_version(_ipex_version) + if torch_major_and_minor != ipex_major_and_minor: + logger.warning( + f"Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*," + f" but PyTorch {_torch_version} is found. Please switch to the matching version and run again." + ) + return False + if min_version: + return version.parse(_ipex_version) >= version.parse(min_version) + return True + + +@lru_cache +def is_torch_xpu_available(check_device=False): + """ + Checks if XPU acceleration is available either via native PyTorch (>=2.6), + `intel_extension_for_pytorch` or via stock PyTorch (>=2.4) and potentially + if a XPU is in the environment. + """ + if not is_torch_available(): + return False + + torch_version = version.parse(_torch_version) + if torch_version.major < 2 or (torch_version.major == 2 and torch_version.minor < 6): + if is_ipex_available(): + import intel_extension_for_pytorch # noqa: F401 + elif torch_version.major < 2 or (torch_version.major == 2 and torch_version.minor < 4): + return False + + import torch + + if check_device: + try: + # Will raise a RuntimeError if no XPU is found + _ = torch.xpu.device_count() + return torch.xpu.is_available() + except RuntimeError: + return False + return hasattr(torch, "xpu") and torch.xpu.is_available() + + +@lru_cache() +def is_bitsandbytes_available(): + if not is_torch_available() or not _bitsandbytes_available: + return False + + import torch + + # `bitsandbytes` versions older than 0.43.1 eagerly require CUDA at import time, + # so those versions of the library are practically only available when CUDA is too. + if version.parse(importlib.metadata.version("bitsandbytes")) < version.parse("0.43.1"): + return torch.cuda.is_available() + + # Newer versions of `bitsandbytes` can be imported on systems without CUDA. + return True + + +def is_bitsandbytes_multi_backend_available() -> bool: + if not is_bitsandbytes_available(): + return False + + import bitsandbytes as bnb + + return "multi_backend" in getattr(bnb, "features", set()) + + +def is_flash_attn_2_available(): + if not is_torch_available(): + return False + + if not _is_package_available("flash_attn"): + return False + + # Let's add an extra check to see if cuda is available + import torch + + if not (torch.cuda.is_available() or is_torch_mlu_available()): + return False + + if torch.version.cuda: + return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.1.0") + elif torch.version.hip: + # TODO: Bump the requirement to 2.1.0 once released in https://github.com/ROCmSoftwarePlatform/flash-attention + return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.0.4") + elif is_torch_mlu_available(): + return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.3.3") + else: + return False + + +@lru_cache() +def is_flash_attn_greater_or_equal_2_10(): + if not _is_package_available("flash_attn"): + return False + + return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.1.0") + + +@lru_cache() +def is_flash_attn_greater_or_equal(library_version: str): + if not _is_package_available("flash_attn"): + return False + + return version.parse(importlib.metadata.version("flash_attn")) >= version.parse(library_version) + + +@lru_cache() +def is_torch_greater_or_equal(library_version: str, accept_dev: bool = False): + """ + Accepts a library version and returns True if the current version of the library is greater than or equal to the + given version. If `accept_dev` is True, it will also accept development versions (e.g. 2.7.0.dev20250320 matches + 2.7.0). + """ + if not _is_package_available("torch"): + return False + + if accept_dev: + return version.parse(version.parse(importlib.metadata.version("torch")).base_version) >= version.parse( + library_version + ) + else: + return version.parse(importlib.metadata.version("torch")) >= version.parse(library_version) + + +def is_torchdistx_available(): + return _torchdistx_available + + +def is_faiss_available(): + return _faiss_available + + +def is_scipy_available(): + return _scipy_available + + +def is_sklearn_available(): + return _sklearn_available + + +def is_sentencepiece_available(): + return _sentencepiece_available + + +def is_seqio_available(): + return _is_seqio_available + + +def is_gguf_available(min_version: str = GGUF_MIN_VERSION): + return _is_gguf_available and version.parse(_gguf_version) >= version.parse(min_version) + + +def is_protobuf_available(): + if importlib.util.find_spec("google") is None: + return False + return importlib.util.find_spec("google.protobuf") is not None + + +def is_fsdp_available(min_version: str = FSDP_MIN_VERSION): + return is_torch_available() and version.parse(_torch_version) >= version.parse(min_version) + + +def is_optimum_available(): + return _optimum_available + + +def is_auto_awq_available(): + return _auto_awq_available + + +def is_optimum_quanto_available(): + # `importlib.metadata.version` doesn't work with `optimum.quanto`, need to put `optimum_quanto` + return _is_optimum_quanto_available + + +def is_quark_available(): + return _quark_available + + +def is_compressed_tensors_available(): + return _compressed_tensors_available + + +def is_auto_gptq_available(): + return _auto_gptq_available + + +def is_gptqmodel_available(): + return _gptqmodel_available + + +def is_eetq_available(): + return _eetq_available + + +def is_fbgemm_gpu_available(): + return _fbgemm_gpu_available + + +def is_levenshtein_available(): + return _levenshtein_available + + +def is_optimum_neuron_available(): + return _optimum_available and _is_package_available("optimum.neuron") + + +def is_safetensors_available(): + return _safetensors_available + + +def is_tokenizers_available(): + return _tokenizers_available + + +@lru_cache +def is_vision_available(): + _pil_available = importlib.util.find_spec("PIL") is not None + if _pil_available: + try: + package_version = importlib.metadata.version("Pillow") + except importlib.metadata.PackageNotFoundError: + try: + package_version = importlib.metadata.version("Pillow-SIMD") + except importlib.metadata.PackageNotFoundError: + return False + logger.debug(f"Detected PIL version {package_version}") + return _pil_available + + +def is_pytesseract_available(): + return _pytesseract_available + + +def is_pytest_available(): + return _pytest_available + + +def is_spacy_available(): + return _spacy_available + + +def is_tensorflow_text_available(): + return is_tf_available() and _tensorflow_text_available + + +def is_keras_nlp_available(): + return is_tensorflow_text_available() and _keras_nlp_available + + +def is_in_notebook(): + try: + # Test adapted from tqdm.autonotebook: https://github.com/tqdm/tqdm/blob/master/tqdm/autonotebook.py + get_ipython = sys.modules["IPython"].get_ipython + if "IPKernelApp" not in get_ipython().config: + raise ImportError("console") + # Removed the lines to include VSCode + if "DATABRICKS_RUNTIME_VERSION" in os.environ and os.environ["DATABRICKS_RUNTIME_VERSION"] < "11.0": + # Databricks Runtime 11.0 and above uses IPython kernel by default so it should be compatible with Jupyter notebook + # https://docs.microsoft.com/en-us/azure/databricks/notebooks/ipython-kernel + raise ImportError("databricks") + + return importlib.util.find_spec("IPython") is not None + except (AttributeError, ImportError, KeyError): + return False + + +def is_pytorch_quantization_available(): + return _pytorch_quantization_available + + +def is_tensorflow_probability_available(): + return _tensorflow_probability_available + + +def is_pandas_available(): + return _pandas_available + + +def is_sagemaker_dp_enabled(): + # Get the sagemaker specific env variable. + sagemaker_params = os.getenv("SM_FRAMEWORK_PARAMS", "{}") + try: + # Parse it and check the field "sagemaker_distributed_dataparallel_enabled". + sagemaker_params = json.loads(sagemaker_params) + if not sagemaker_params.get("sagemaker_distributed_dataparallel_enabled", False): + return False + except json.JSONDecodeError: + return False + # Lastly, check if the `smdistributed` module is present. + return _smdistributed_available + + +def is_sagemaker_mp_enabled(): + # Get the sagemaker specific mp parameters from smp_options variable. + smp_options = os.getenv("SM_HP_MP_PARAMETERS", "{}") + try: + # Parse it and check the field "partitions" is included, it is required for model parallel. + smp_options = json.loads(smp_options) + if "partitions" not in smp_options: + return False + except json.JSONDecodeError: + return False + + # Get the sagemaker specific framework parameters from mpi_options variable. + mpi_options = os.getenv("SM_FRAMEWORK_PARAMS", "{}") + try: + # Parse it and check the field "sagemaker_distributed_dataparallel_enabled". + mpi_options = json.loads(mpi_options) + if not mpi_options.get("sagemaker_mpi_enabled", False): + return False + except json.JSONDecodeError: + return False + # Lastly, check if the `smdistributed` module is present. + return _smdistributed_available + + +def is_training_run_on_sagemaker(): + return "SAGEMAKER_JOB_NAME" in os.environ + + +def is_soundfile_available(): + return _soundfile_available + + +def is_timm_available(): + return _timm_available + + +def is_natten_available(): + return _natten_available + + +def is_nltk_available(): + return _nltk_available + + +def is_torchaudio_available(): + return _torchaudio_available + + +def is_torchao_available(min_version: str = TORCHAO_MIN_VERSION): + return _torchao_available and version.parse(_torchao_version) >= version.parse(min_version) + + +def is_speech_available(): + # For now this depends on torchaudio but the exact dependency might evolve in the future. + return _torchaudio_available + + +def is_spqr_available(): + return _spqr_available + + +def is_phonemizer_available(): + return _phonemizer_available + + +def is_uroman_available(): + return _uroman_available + + +def torch_only_method(fn): + def wrapper(*args, **kwargs): + if not _torch_available: + raise ImportError( + "You need to install pytorch to use this method or class, " + "or activate it with environment variables USE_TORCH=1 and USE_TF=0." + ) + else: + return fn(*args, **kwargs) + + return wrapper + + +def is_ccl_available(): + return _is_ccl_available + + +def is_sudachi_available(): + return _sudachipy_available + + +def get_sudachi_version(): + return _sudachipy_version + + +def is_sudachi_projection_available(): + if not is_sudachi_available(): + return False + + # NOTE: We require sudachipy>=0.6.8 to use projection option in sudachi_kwargs for the constructor of BertJapaneseTokenizer. + # - `projection` option is not supported in sudachipy<0.6.8, see https://github.com/WorksApplications/sudachi.rs/issues/230 + return version.parse(_sudachipy_version) >= version.parse("0.6.8") + + +def is_jumanpp_available(): + return (importlib.util.find_spec("rhoknp") is not None) and (shutil.which("jumanpp") is not None) + + +def is_cython_available(): + return importlib.util.find_spec("pyximport") is not None + + +def is_jieba_available(): + return _jieba_available + + +def is_jinja_available(): + return _jinja_available + + +def is_mlx_available(): + return _mlx_available + + +def is_num2words_available(): + return _num2words_available + + +def is_tiktoken_available(): + return _tiktoken_available and _blobfile_available + + +def is_liger_kernel_available(): + if not _liger_kernel_available: + return False + + return version.parse(importlib.metadata.version("liger_kernel")) >= version.parse("0.3.0") + + +def is_triton_available(): + return _triton_available + + +def is_rich_available(): + return _rich_available + + +# docstyle-ignore +AV_IMPORT_ERROR = """ +{0} requires the PyAv library but it was not found in your environment. You can install it with: +``` +pip install av +``` +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +YT_DLP_IMPORT_ERROR = """ +{0} requires the YT-DLP library but it was not found in your environment. You can install it with: +``` +pip install yt-dlp +``` +Please note that you may need to restart your runtime after installation. +""" + +DECORD_IMPORT_ERROR = """ +{0} requires the PyAv library but it was not found in your environment. You can install it with: +``` +pip install decord +``` +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +CV2_IMPORT_ERROR = """ +{0} requires the OpenCV library but it was not found in your environment. You can install it with: +``` +pip install opencv-python +``` +Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +DATASETS_IMPORT_ERROR = """ +{0} requires the 🤗 Datasets library but it was not found in your environment. You can install it with: +``` +pip install datasets +``` +In a notebook or a colab, you can install it by executing a cell with +``` +!pip install datasets +``` +then restarting your kernel. + +Note that if you have a local folder named `datasets` or a local python file named `datasets.py` in your current +working directory, python may try to import this instead of the 🤗 Datasets library. You should rename this folder or +that python file if that's the case. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +TOKENIZERS_IMPORT_ERROR = """ +{0} requires the 🤗 Tokenizers library but it was not found in your environment. You can install it with: +``` +pip install tokenizers +``` +In a notebook or a colab, you can install it by executing a cell with +``` +!pip install tokenizers +``` +Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +SENTENCEPIECE_IMPORT_ERROR = """ +{0} requires the SentencePiece library but it was not found in your environment. Checkout the instructions on the +installation page of its repo: https://github.com/google/sentencepiece#installation and follow the ones +that match your environment. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +PROTOBUF_IMPORT_ERROR = """ +{0} requires the protobuf library but it was not found in your environment. Checkout the instructions on the +installation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones +that match your environment. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +FAISS_IMPORT_ERROR = """ +{0} requires the faiss library but it was not found in your environment. Checkout the instructions on the +installation page of its repo: https://github.com/facebookresearch/faiss/blob/master/INSTALL.md and follow the ones +that match your environment. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +PYTORCH_IMPORT_ERROR = """ +{0} requires the PyTorch library but it was not found in your environment. Checkout the instructions on the +installation page: https://pytorch.org/get-started/locally/ and follow the ones that match your environment. +Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +TORCHVISION_IMPORT_ERROR = """ +{0} requires the Torchvision library but it was not found in your environment. Checkout the instructions on the +installation page: https://pytorch.org/get-started/locally/ and follow the ones that match your environment. +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +PYTORCH_IMPORT_ERROR_WITH_TF = """ +{0} requires the PyTorch library but it was not found in your environment. +However, we were able to find a TensorFlow installation. TensorFlow classes begin +with "TF", but are otherwise identically named to our PyTorch classes. This +means that the TF equivalent of the class you tried to import would be "TF{0}". +If you want to use TensorFlow, please use TF classes instead! + +If you really do want to use PyTorch please go to +https://pytorch.org/get-started/locally/ and follow the instructions that +match your environment. +""" + +# docstyle-ignore +TF_IMPORT_ERROR_WITH_PYTORCH = """ +{0} requires the TensorFlow library but it was not found in your environment. +However, we were able to find a PyTorch installation. PyTorch classes do not begin +with "TF", but are otherwise identically named to our TF classes. +If you want to use PyTorch, please use those classes instead! + +If you really do want to use TensorFlow, please follow the instructions on the +installation page https://www.tensorflow.org/install that match your environment. +""" + +# docstyle-ignore +BS4_IMPORT_ERROR = """ +{0} requires the Beautiful Soup library but it was not found in your environment. You can install it with pip: +`pip install beautifulsoup4`. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +SKLEARN_IMPORT_ERROR = """ +{0} requires the scikit-learn library but it was not found in your environment. You can install it with: +``` +pip install -U scikit-learn +``` +In a notebook or a colab, you can install it by executing a cell with +``` +!pip install -U scikit-learn +``` +Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +TENSORFLOW_IMPORT_ERROR = """ +{0} requires the TensorFlow library but it was not found in your environment. Checkout the instructions on the +installation page: https://www.tensorflow.org/install and follow the ones that match your environment. +Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +DETECTRON2_IMPORT_ERROR = """ +{0} requires the detectron2 library but it was not found in your environment. Checkout the instructions on the +installation page: https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md and follow the ones +that match your environment. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +FLAX_IMPORT_ERROR = """ +{0} requires the FLAX library but it was not found in your environment. Checkout the instructions on the +installation page: https://github.com/google/flax and follow the ones that match your environment. +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +FTFY_IMPORT_ERROR = """ +{0} requires the ftfy library but it was not found in your environment. Checkout the instructions on the +installation section: https://github.com/rspeer/python-ftfy/tree/master#installing and follow the ones +that match your environment. Please note that you may need to restart your runtime after installation. +""" + +LEVENSHTEIN_IMPORT_ERROR = """ +{0} requires the python-Levenshtein library but it was not found in your environment. You can install it with pip: `pip +install python-Levenshtein`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +G2P_EN_IMPORT_ERROR = """ +{0} requires the g2p-en library but it was not found in your environment. You can install it with pip: +`pip install g2p-en`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +PYTORCH_QUANTIZATION_IMPORT_ERROR = """ +{0} requires the pytorch-quantization library but it was not found in your environment. You can install it with pip: +`pip install pytorch-quantization --extra-index-url https://pypi.ngc.nvidia.com` +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +TENSORFLOW_PROBABILITY_IMPORT_ERROR = """ +{0} requires the tensorflow_probability library but it was not found in your environment. You can install it with pip as +explained here: https://github.com/tensorflow/probability. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +TENSORFLOW_TEXT_IMPORT_ERROR = """ +{0} requires the tensorflow_text library but it was not found in your environment. You can install it with pip as +explained here: https://www.tensorflow.org/text/guide/tf_text_intro. +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +TORCHAUDIO_IMPORT_ERROR = """ +{0} requires the torchaudio library but it was not found in your environment. Please install it and restart your +runtime. +""" + +# docstyle-ignore +PANDAS_IMPORT_ERROR = """ +{0} requires the pandas library but it was not found in your environment. You can install it with pip as +explained here: https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html. +Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +PHONEMIZER_IMPORT_ERROR = """ +{0} requires the phonemizer library but it was not found in your environment. You can install it with pip: +`pip install phonemizer`. Please note that you may need to restart your runtime after installation. +""" +# docstyle-ignore +UROMAN_IMPORT_ERROR = """ +{0} requires the uroman library but it was not found in your environment. You can install it with pip: +`pip install uroman`. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +SACREMOSES_IMPORT_ERROR = """ +{0} requires the sacremoses library but it was not found in your environment. You can install it with pip: +`pip install sacremoses`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +SCIPY_IMPORT_ERROR = """ +{0} requires the scipy library but it was not found in your environment. You can install it with pip: +`pip install scipy`. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +SPEECH_IMPORT_ERROR = """ +{0} requires the torchaudio library but it was not found in your environment. You can install it with pip: +`pip install torchaudio`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +TIMM_IMPORT_ERROR = """ +{0} requires the timm library but it was not found in your environment. You can install it with pip: +`pip install timm`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +NATTEN_IMPORT_ERROR = """ +{0} requires the natten library but it was not found in your environment. You can install it by referring to: +shi-labs.com/natten . You can also install it with pip (may take longer to build): +`pip install natten`. Please note that you may need to restart your runtime after installation. +""" + +NUMEXPR_IMPORT_ERROR = """ +{0} requires the numexpr library but it was not found in your environment. You can install it by referring to: +https://numexpr.readthedocs.io/en/latest/index.html. +""" + + +# docstyle-ignore +NLTK_IMPORT_ERROR = """ +{0} requires the NLTK library but it was not found in your environment. You can install it by referring to: +https://www.nltk.org/install.html. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +VISION_IMPORT_ERROR = """ +{0} requires the PIL library but it was not found in your environment. You can install it with pip: +`pip install pillow`. Please note that you may need to restart your runtime after installation. +""" + + +# docstyle-ignore +PYTESSERACT_IMPORT_ERROR = """ +{0} requires the PyTesseract library but it was not found in your environment. You can install it with pip: +`pip install pytesseract`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +PYCTCDECODE_IMPORT_ERROR = """ +{0} requires the pyctcdecode library but it was not found in your environment. You can install it with pip: +`pip install pyctcdecode`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +ACCELERATE_IMPORT_ERROR = """ +{0} requires the accelerate library >= {ACCELERATE_MIN_VERSION} it was not found in your environment. +You can install or update it with pip: `pip install --upgrade accelerate`. Please note that you may need to restart your +runtime after installation. +""" + +# docstyle-ignore +CCL_IMPORT_ERROR = """ +{0} requires the torch ccl library but it was not found in your environment. You can install it with pip: +`pip install oneccl_bind_pt -f https://developer.intel.com/ipex-whl-stable` +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +ESSENTIA_IMPORT_ERROR = """ +{0} requires essentia library. But that was not found in your environment. You can install them with pip: +`pip install essentia==2.1b6.dev1034` +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +LIBROSA_IMPORT_ERROR = """ +{0} requires thes librosa library. But that was not found in your environment. You can install them with pip: +`pip install librosa` +Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +PRETTY_MIDI_IMPORT_ERROR = """ +{0} requires thes pretty_midi library. But that was not found in your environment. You can install them with pip: +`pip install pretty_midi` +Please note that you may need to restart your runtime after installation. +""" + + +CYTHON_IMPORT_ERROR = """ +{0} requires the Cython library but it was not found in your environment. You can install it with pip: `pip install +Cython`. Please note that you may need to restart your runtime after installation. +""" + +JIEBA_IMPORT_ERROR = """ +{0} requires the jieba library but it was not found in your environment. You can install it with pip: `pip install +jieba`. Please note that you may need to restart your runtime after installation. +""" + +PEFT_IMPORT_ERROR = """ +{0} requires the peft library but it was not found in your environment. You can install it with pip: `pip install +peft`. Please note that you may need to restart your runtime after installation. +""" + +JINJA_IMPORT_ERROR = """ +{0} requires the jinja library but it was not found in your environment. You can install it with pip: `pip install +jinja2`. Please note that you may need to restart your runtime after installation. +""" + +RICH_IMPORT_ERROR = """ +{0} requires the rich library but it was not found in your environment. You can install it with pip: `pip install +rich`. Please note that you may need to restart your runtime after installation. +""" + +BACKENDS_MAPPING = OrderedDict( + [ + ("av", (is_av_available, AV_IMPORT_ERROR)), + ("bs4", (is_bs4_available, BS4_IMPORT_ERROR)), + ("cv2", (is_cv2_available, CV2_IMPORT_ERROR)), + ("datasets", (is_datasets_available, DATASETS_IMPORT_ERROR)), + ("decord", (is_decord_available, DECORD_IMPORT_ERROR)), + ("detectron2", (is_detectron2_available, DETECTRON2_IMPORT_ERROR)), + ("essentia", (is_essentia_available, ESSENTIA_IMPORT_ERROR)), + ("faiss", (is_faiss_available, FAISS_IMPORT_ERROR)), + ("flax", (is_flax_available, FLAX_IMPORT_ERROR)), + ("ftfy", (is_ftfy_available, FTFY_IMPORT_ERROR)), + ("g2p_en", (is_g2p_en_available, G2P_EN_IMPORT_ERROR)), + ("pandas", (is_pandas_available, PANDAS_IMPORT_ERROR)), + ("phonemizer", (is_phonemizer_available, PHONEMIZER_IMPORT_ERROR)), + ("uroman", (is_uroman_available, UROMAN_IMPORT_ERROR)), + ("pretty_midi", (is_pretty_midi_available, PRETTY_MIDI_IMPORT_ERROR)), + ("levenshtein", (is_levenshtein_available, LEVENSHTEIN_IMPORT_ERROR)), + ("librosa", (is_librosa_available, LIBROSA_IMPORT_ERROR)), + ("protobuf", (is_protobuf_available, PROTOBUF_IMPORT_ERROR)), + ("pyctcdecode", (is_pyctcdecode_available, PYCTCDECODE_IMPORT_ERROR)), + ("pytesseract", (is_pytesseract_available, PYTESSERACT_IMPORT_ERROR)), + ("sacremoses", (is_sacremoses_available, SACREMOSES_IMPORT_ERROR)), + ("pytorch_quantization", (is_pytorch_quantization_available, PYTORCH_QUANTIZATION_IMPORT_ERROR)), + ("sentencepiece", (is_sentencepiece_available, SENTENCEPIECE_IMPORT_ERROR)), + ("sklearn", (is_sklearn_available, SKLEARN_IMPORT_ERROR)), + ("speech", (is_speech_available, SPEECH_IMPORT_ERROR)), + ("tensorflow_probability", (is_tensorflow_probability_available, TENSORFLOW_PROBABILITY_IMPORT_ERROR)), + ("tf", (is_tf_available, TENSORFLOW_IMPORT_ERROR)), + ("tensorflow_text", (is_tensorflow_text_available, TENSORFLOW_TEXT_IMPORT_ERROR)), + ("timm", (is_timm_available, TIMM_IMPORT_ERROR)), + ("torchaudio", (is_torchaudio_available, TORCHAUDIO_IMPORT_ERROR)), + ("natten", (is_natten_available, NATTEN_IMPORT_ERROR)), + ("nltk", (is_nltk_available, NLTK_IMPORT_ERROR)), + ("tokenizers", (is_tokenizers_available, TOKENIZERS_IMPORT_ERROR)), + ("torch", (is_torch_available, PYTORCH_IMPORT_ERROR)), + ("torchvision", (is_torchvision_available, TORCHVISION_IMPORT_ERROR)), + ("vision", (is_vision_available, VISION_IMPORT_ERROR)), + ("scipy", (is_scipy_available, SCIPY_IMPORT_ERROR)), + ("accelerate", (is_accelerate_available, ACCELERATE_IMPORT_ERROR)), + ("oneccl_bind_pt", (is_ccl_available, CCL_IMPORT_ERROR)), + ("cython", (is_cython_available, CYTHON_IMPORT_ERROR)), + ("jieba", (is_jieba_available, JIEBA_IMPORT_ERROR)), + ("peft", (is_peft_available, PEFT_IMPORT_ERROR)), + ("jinja", (is_jinja_available, JINJA_IMPORT_ERROR)), + ("yt_dlp", (is_yt_dlp_available, YT_DLP_IMPORT_ERROR)), + ("rich", (is_rich_available, RICH_IMPORT_ERROR)), + ] +) + + +def requires_backends(obj, backends): + if not isinstance(backends, (list, tuple)): + backends = [backends] + + name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__ + + # Raise an error for users who might not realize that classes without "TF" are torch-only + if "torch" in backends and "tf" not in backends and not is_torch_available() and is_tf_available(): + raise ImportError(PYTORCH_IMPORT_ERROR_WITH_TF.format(name)) + + # Raise the inverse error for PyTorch users trying to load TF classes + if "tf" in backends and "torch" not in backends and is_torch_available() and not is_tf_available(): + raise ImportError(TF_IMPORT_ERROR_WITH_PYTORCH.format(name)) + + checks = (BACKENDS_MAPPING[backend] for backend in backends) + failed = [msg.format(name) for available, msg in checks if not available()] + if failed: + raise ImportError("".join(failed)) + + +class DummyObject(type): + """ + Metaclass for the dummy objects. Any class inheriting from it will return the ImportError generated by + `requires_backend` each time a user tries to access any method of that class. + """ + + def __getattribute__(cls, key): + if key.startswith("_") and key != "_from_config": + return super().__getattribute__(key) + requires_backends(cls, cls._backends) + + +def is_torch_fx_proxy(x): + if is_torch_fx_available(): + import torch.fx + + return isinstance(x, torch.fx.Proxy) + return False + + +BACKENDS_T = FrozenSet[str] +IMPORT_STRUCTURE_T = Dict[BACKENDS_T, Dict[str, Set[str]]] + + +class _LazyModule(ModuleType): + """ + Module class that surfaces all objects but only performs associated imports when the objects are requested. + """ + + # Very heavily inspired by optuna.integration._IntegrationModule + # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py + def __init__( + self, + name: str, + module_file: str, + import_structure: IMPORT_STRUCTURE_T, + module_spec: Optional[importlib.machinery.ModuleSpec] = None, + extra_objects: Dict[str, object] = None, + ): + super().__init__(name) + + self._object_missing_backend = {} + if any(isinstance(key, frozenset) for key in import_structure.keys()): + self._modules = set() + self._class_to_module = {} + self.__all__ = [] + + _import_structure = {} + + for backends, module in import_structure.items(): + missing_backends = [] + for backend in backends: + if backend not in BACKENDS_MAPPING: + raise ValueError( + f"Error: the following backend: '{backend}' was specified around object {module} but isn't specified in the backends mapping." + ) + callable, error = BACKENDS_MAPPING[backend] + if not callable(): + missing_backends.append(backend) + self._modules = self._modules.union(set(module.keys())) + + for key, values in module.items(): + if len(missing_backends): + self._object_missing_backend[key] = missing_backends + + for value in values: + self._class_to_module[value] = key + if len(missing_backends): + self._object_missing_backend[value] = missing_backends + _import_structure.setdefault(key, []).extend(values) + + # Needed for autocompletion in an IDE + self.__all__.extend(list(module.keys()) + list(chain(*module.values()))) + + self.__file__ = module_file + self.__spec__ = module_spec + self.__path__ = [os.path.dirname(module_file)] + self._objects = {} if extra_objects is None else extra_objects + self._name = name + self._import_structure = _import_structure + + # This can be removed once every exportable object has a `export()` export. + else: + self._modules = set(import_structure.keys()) + self._class_to_module = {} + for key, values in import_structure.items(): + for value in values: + self._class_to_module[value] = key + # Needed for autocompletion in an IDE + self.__all__ = list(import_structure.keys()) + list(chain(*import_structure.values())) + self.__file__ = module_file + self.__spec__ = module_spec + self.__path__ = [os.path.dirname(module_file)] + self._objects = {} if extra_objects is None else extra_objects + self._name = name + self._import_structure = import_structure + + # Needed for autocompletion in an IDE + def __dir__(self): + result = super().__dir__() + # The elements of self.__all__ that are submodules may or may not be in the dir already, depending on whether + # they have been accessed or not. So we only add the elements of self.__all__ that are not already in the dir. + for attr in self.__all__: + if attr not in result: + result.append(attr) + return result + + def __getattr__(self, name: str) -> Any: + if name in self._objects: + return self._objects[name] + if name in self._object_missing_backend.keys(): + missing_backends = self._object_missing_backend[name] + + class Placeholder(metaclass=DummyObject): + _backends = missing_backends + + def __init__(self, *args, **kwargs): + requires_backends(self, missing_backends) + + Placeholder.__name__ = name + Placeholder.__module__ = self.__spec__ + + value = Placeholder + elif name in self._class_to_module.keys(): + module = self._get_module(self._class_to_module[name]) + value = getattr(module, name) + elif name in self._modules: + value = self._get_module(name) + else: + raise AttributeError(f"module {self.__name__} has no attribute {name}") + + setattr(self, name, value) + return value + + def _get_module(self, module_name: str): + try: + return importlib.import_module("." + module_name, self.__name__) + except Exception as e: + raise RuntimeError( + f"Failed to import {self.__name__}.{module_name} because of the following error (look up to see its" + f" traceback):\n{e}" + ) from e + + def __reduce__(self): + return (self.__class__, (self._name, self.__file__, self._import_structure)) + + +class OptionalDependencyNotAvailable(BaseException): + """Internally used error class for signalling an optional dependency was not found.""" + + +def direct_transformers_import(path: str, file="__init__.py") -> ModuleType: + """Imports transformers directly + + Args: + path (`str`): The path to the source file + file (`str`, *optional*): The file to join with the path. Defaults to "__init__.py". + + Returns: + `ModuleType`: The resulting imported module + """ + name = "transformers" + location = os.path.join(path, file) + spec = importlib.util.spec_from_file_location(name, location, submodule_search_locations=[path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module = sys.modules[name] + return module + + +def export(*, backends=()): + """ + This decorator enables two things: + - Attaching a `__backends` tuple to an object to see what are the necessary backends for it + to execute correctly without instantiating it + - The '@export' string is used to dynamically import objects + """ + for backend in backends: + if backend not in BACKENDS_MAPPING: + raise ValueError(f"Backend should be defined in the BACKENDS_MAPPING. Offending backend: {backend}") + + if not isinstance(backends, tuple): + raise ValueError("Backends should be a tuple.") + + def inner_fn(fun): + fun.__backends = backends + return fun + + return inner_fn + + +BASE_FILE_REQUIREMENTS = { + lambda e: "modeling_tf_" in e: ("tf",), + lambda e: "modeling_flax_" in e: ("flax",), + lambda e: "modeling_" in e: ("torch",), + lambda e: e.startswith("tokenization_") and e.endswith("_fast"): ("tokenizers",), +} + + +def fetch__all__(file_content): + """ + Returns the content of the __all__ variable in the file content. + Returns None if not defined, otherwise returns a list of strings. + """ + + if "__all__" not in file_content: + return [] + + start_index = None + lines = file_content.splitlines() + for index, line in enumerate(lines): + if line.startswith("__all__"): + start_index = index + + # There is no line starting with `__all__` + if start_index is None: + return [] + + lines = lines[start_index:] + + if not lines[0].startswith("__all__"): + raise ValueError( + "fetch__all__ accepts a list of lines, with the first line being the __all__ variable declaration" + ) + + # __all__ is defined on a single line + if lines[0].endswith("]"): + return [obj.strip("\"' ") for obj in lines[0].split("=")[1].strip(" []").split(",")] + + # __all__ is defined on multiple lines + else: + _all = [] + for __all__line_index in range(1, len(lines)): + if lines[__all__line_index].strip() == "]": + return _all + else: + _all.append(lines[__all__line_index].strip("\"', ")) + + return _all + + +@lru_cache() +def create_import_structure_from_path(module_path): + """ + This method takes the path to a file/a folder and returns the import structure. + If a file is given, it will return the import structure of the parent folder. + + Import structures are designed to be digestible by `_LazyModule` objects. They are + created from the __all__ definitions in each files as well as the `@export` decorators + above methods and objects. + + The import structure allows explicit display of the required backends for a given object. + These backends are specified in two ways: + + 1. Through their `@export`, if they are exported with that decorator. This `@export` decorator + accepts a `backend` tuple kwarg mentioning which backends are required to run this object. + + 2. If an object is defined in a file with "default" backends, it will have, at a minimum, this + backend specified. The default backends are defined according to the filename: + + - If a file is named like `modeling_*.py`, it will have a `torch` backend + - If a file is named like `modeling_tf_*.py`, it will have a `tf` backend + - If a file is named like `modeling_flax_*.py`, it will have a `flax` backend + - If a file is named like `tokenization_*_fast.py`, it will have a `tokenizers` backend + + Backends serve the purpose of displaying a clear error message to the user in case the backends are not installed. + Should an object be imported without its required backends being in the environment, any attempt to use the + object will raise an error mentioning which backend(s) should be added to the environment in order to use + that object. + + Here's an example of an input import structure at the src.transformers.models level: + + { + 'albert': { + frozenset(): { + 'configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'} + }, + frozenset({'tokenizers'}): { + 'tokenization_albert_fast': {'AlbertTokenizerFast'} + }, + }, + 'align': { + frozenset(): { + 'configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'}, + 'processing_align': {'AlignProcessor'} + }, + }, + 'altclip': { + frozenset(): { + 'configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'}, + 'processing_altclip': {'AltCLIPProcessor'}, + } + } + } + """ + import_structure = {} + if os.path.isdir(module_path): + directory = module_path + adjacent_modules = [] + + for f in os.listdir(module_path): + if f != "__pycache__" and os.path.isdir(os.path.join(module_path, f)): + import_structure[f] = create_import_structure_from_path(os.path.join(module_path, f)) + + elif not os.path.isdir(os.path.join(directory, f)): + adjacent_modules.append(f) + + else: + directory = os.path.dirname(module_path) + adjacent_modules = [f for f in os.listdir(directory) if not os.path.isdir(os.path.join(directory, f))] + + # We're only taking a look at files different from __init__.py + # We could theoretically export things directly from the __init__.py + # files, but this is not supported at this time. + if "__init__.py" in adjacent_modules: + adjacent_modules.remove("__init__.py") + + # Modular files should not be imported + def find_substring(substring, list_): + return any(substring in x for x in list_) + + if find_substring("modular_", adjacent_modules) and find_substring("modeling_", adjacent_modules): + adjacent_modules = [module for module in adjacent_modules if "modular_" not in module] + + module_requirements = {} + for module_name in adjacent_modules: + # Only modules ending in `.py` are accepted here. + if not module_name.endswith(".py"): + continue + + with open(os.path.join(directory, module_name), encoding="utf-8") as f: + file_content = f.read() + + # Remove the .py suffix + module_name = module_name[:-3] + + previous_line = "" + previous_index = 0 + + # Some files have some requirements by default. + # For example, any file named `modeling_tf_xxx.py` + # should have TensorFlow as a required backend. + base_requirements = () + for string_check, requirements in BASE_FILE_REQUIREMENTS.items(): + if string_check(module_name): + base_requirements = requirements + break + + # Objects that have a `@export` assigned to them will get exported + # with the backends specified in the decorator as well as the file backends. + exported_objects = set() + if "@export" in file_content: + lines = file_content.split("\n") + for index, line in enumerate(lines): + # This allows exporting items with other decorators. We'll take a look + # at the line that follows at the same indentation level. + if line.startswith((" ", "\t", "@", ")")) and not line.startswith("@export"): + continue + + # Skipping line enables putting whatever we want between the + # export() call and the actual class/method definition. + # This is what enables having # Copied from statements, docs, etc. + skip_line = False + + if "@export" in previous_line: + skip_line = False + + # Backends are defined on the same line as export + if "backends" in previous_line: + backends_string = previous_line.split("backends=")[1].split("(")[1].split(")")[0] + backends = tuple(sorted([b.strip("'\",") for b in backends_string.split(", ") if b])) + + # Backends are defined in the lines following export, for example such as: + # @export( + # backends=( + # "sentencepiece", + # "torch", + # "tf", + # ) + # ) + # + # or + # + # @export( + # backends=( + # "sentencepiece", "tf" + # ) + # ) + elif "backends" in lines[previous_index + 1]: + backends = [] + for backend_line in lines[previous_index:index]: + if "backends" in backend_line: + backend_line = backend_line.split("=")[1] + if '"' in backend_line or "'" in backend_line: + if ", " in backend_line: + backends.extend(backend.strip("()\"', ") for backend in backend_line.split(", ")) + else: + backends.append(backend_line.strip("()\"', ")) + + # If the line is only a ')', then we reached the end of the backends and we break. + if backend_line.strip() == ")": + break + backends = tuple(backends) + + # No backends are registered for export + else: + backends = () + + backends = frozenset(backends + base_requirements) + if backends not in module_requirements: + module_requirements[backends] = {} + if module_name not in module_requirements[backends]: + module_requirements[backends][module_name] = set() + + if not line.startswith("class") and not line.startswith("def"): + skip_line = True + else: + start_index = 6 if line.startswith("class") else 4 + object_name = line[start_index:].split("(")[0].strip(":") + module_requirements[backends][module_name].add(object_name) + exported_objects.add(object_name) + + if not skip_line: + previous_line = line + previous_index = index + + # All objects that are in __all__ should be exported by default. + # These objects are exported with the file backends. + if "__all__" in file_content: + for _all_object in fetch__all__(file_content): + if _all_object not in exported_objects: + backends = frozenset(base_requirements) + if backends not in module_requirements: + module_requirements[backends] = {} + if module_name not in module_requirements[backends]: + module_requirements[backends][module_name] = set() + + module_requirements[backends][module_name].add(_all_object) + + import_structure = {**module_requirements, **import_structure} + return import_structure + + +def spread_import_structure(nested_import_structure): + """ + This method takes as input an unordered import structure and brings the required backends at the top-level, + aggregating modules and objects under their required backends. + + Here's an example of an input import structure at the src.transformers.models level: + + { + 'albert': { + frozenset(): { + 'configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'} + }, + frozenset({'tokenizers'}): { + 'tokenization_albert_fast': {'AlbertTokenizerFast'} + }, + }, + 'align': { + frozenset(): { + 'configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'}, + 'processing_align': {'AlignProcessor'} + }, + }, + 'altclip': { + frozenset(): { + 'configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'}, + 'processing_altclip': {'AltCLIPProcessor'}, + } + } + } + + Here's an example of an output import structure at the src.transformers.models level: + + { + frozenset({'tokenizers'}): { + 'albert.tokenization_albert_fast': {'AlbertTokenizerFast'} + }, + frozenset(): { + 'albert.configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'}, + 'align.processing_align': {'AlignProcessor'}, + 'align.configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'}, + 'altclip.configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'}, + 'altclip.processing_altclip': {'AltCLIPProcessor'} + } + } + + """ + + def propagate_frozenset(unordered_import_structure): + tuple_first_import_structure = {} + for _key, _value in unordered_import_structure.items(): + if not isinstance(_value, dict): + tuple_first_import_structure[_key] = _value + + elif any(isinstance(v, frozenset) for v in _value.keys()): + # Here we want to switch around key and v + for k, v in _value.items(): + if isinstance(k, frozenset): + if k not in tuple_first_import_structure: + tuple_first_import_structure[k] = {} + tuple_first_import_structure[k][_key] = v + + else: + tuple_first_import_structure[_key] = propagate_frozenset(_value) + + return tuple_first_import_structure + + def flatten_dict(_dict, previous_key=None): + items = [] + for _key, _value in _dict.items(): + _key = f"{previous_key}.{_key}" if previous_key is not None else _key + if isinstance(_value, dict): + items.extend(flatten_dict(_value, _key).items()) + else: + items.append((_key, _value)) + return dict(items) + + # The tuples contain the necessary backends. We want these first, so we propagate them up the + # import structure. + ordered_import_structure = nested_import_structure + + # 6 is a number that gives us sufficient depth to go through all files and foreseeable folder depths + # while not taking too long to parse. + for i in range(6): + ordered_import_structure = propagate_frozenset(ordered_import_structure) + + # We then flatten the dict so that it references a module path. + flattened_import_structure = {} + for key, value in ordered_import_structure.copy().items(): + if isinstance(key, str): + del ordered_import_structure[key] + else: + flattened_import_structure[key] = flatten_dict(value) + + return flattened_import_structure + + +def define_import_structure(module_path: str) -> IMPORT_STRUCTURE_T: + """ + This method takes a module_path as input and creates an import structure digestible by a _LazyModule. + + Here's an example of an output import structure at the src.transformers.models level: + + { + frozenset({'tokenizers'}): { + 'albert.tokenization_albert_fast': {'AlbertTokenizerFast'} + }, + frozenset(): { + 'albert.configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'}, + 'align.processing_align': {'AlignProcessor'}, + 'align.configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'}, + 'altclip.configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'}, + 'altclip.processing_altclip': {'AltCLIPProcessor'} + } + } + + The import structure is a dict defined with frozensets as keys, and dicts of strings to sets of objects. + """ + import_structure = create_import_structure_from_path(module_path) + return spread_import_structure(import_structure) + + +def clear_import_cache(): + """ + Clear cached Transformers modules to allow reloading modified code. + + This is useful when actively developing/modifying Transformers code. + """ + # Get all transformers modules + transformers_modules = [mod_name for mod_name in sys.modules if mod_name.startswith("transformers.")] + + # Remove them from sys.modules + for mod_name in transformers_modules: + module = sys.modules[mod_name] + # Clear _LazyModule caches if applicable + if isinstance(module, _LazyModule): + module._objects = {} # Clear cached objects + del sys.modules[mod_name] + + # Force reload main transformers module + if "transformers" in sys.modules: + main_module = sys.modules["transformers"] + if isinstance(main_module, _LazyModule): + main_module._objects = {} # Clear cached objects + importlib.reload(main_module) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/integration_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/integration_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a696612c3bfd784107421f28eb227e3763ed514b --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/integration_utils.py @@ -0,0 +1,2348 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Integrations with other Python libraries. +""" + +import functools +import importlib.metadata +import importlib.util +import json +import numbers +import os +import pickle +import shutil +import sys +import tempfile +from dataclasses import asdict, fields +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Union + +import numpy as np +import packaging.version + +from .. import PreTrainedModel, TFPreTrainedModel +from .. import __version__ as version +from ..utils import ( + PushToHubMixin, + flatten_dict, + is_datasets_available, + is_pandas_available, + is_tf_available, + is_torch_available, + logging, +) + + +logger = logging.get_logger(__name__) + +if is_torch_available(): + import torch + +# comet_ml requires to be imported before any ML frameworks +_MIN_COMET_VERSION = "3.43.2" +try: + _comet_version = importlib.metadata.version("comet_ml") + _is_comet_installed = True + + _is_comet_recent_enough = packaging.version.parse(_comet_version) >= packaging.version.parse(_MIN_COMET_VERSION) + + # Check if the Comet API Key is set + import comet_ml + + if comet_ml.config.get_config("comet.api_key") is not None: + _is_comet_configured = True + else: + _is_comet_configured = False +except (importlib.metadata.PackageNotFoundError, ImportError, ValueError, TypeError, AttributeError, KeyError): + _comet_version = None + _is_comet_installed = False + _is_comet_recent_enough = False + _is_comet_configured = False + +_has_neptune = ( + importlib.util.find_spec("neptune") is not None or importlib.util.find_spec("neptune-client") is not None +) +if TYPE_CHECKING and _has_neptune: + try: + _neptune_version = importlib.metadata.version("neptune") + logger.info(f"Neptune version {_neptune_version} available.") + except importlib.metadata.PackageNotFoundError: + try: + _neptune_version = importlib.metadata.version("neptune-client") + logger.info(f"Neptune-client version {_neptune_version} available.") + except importlib.metadata.PackageNotFoundError: + _has_neptune = False + +from .. import modelcard # noqa: E402 +from ..trainer_callback import ProgressCallback, TrainerCallback # noqa: E402 +from ..trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun, IntervalStrategy # noqa: E402 +from ..training_args import ParallelMode # noqa: E402 +from ..utils import ENV_VARS_TRUE_VALUES, is_torch_xla_available # noqa: E402 + + +# Integration functions: +def is_wandb_available(): + # any value of WANDB_DISABLED disables wandb + if os.getenv("WANDB_DISABLED", "").upper() in ENV_VARS_TRUE_VALUES: + logger.warning( + "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the " + "--report_to flag to control the integrations used for logging result (for instance --report_to none)." + ) + return False + return importlib.util.find_spec("wandb") is not None + + +def is_clearml_available(): + return importlib.util.find_spec("clearml") is not None + + +def is_comet_available(): + if os.getenv("COMET_MODE", "").upper() == "DISABLED": + logger.warning( + "Using the `COMET_MODE=DISABLED` environment variable is deprecated and will be removed in v5. Use the " + "--report_to flag to control the integrations used for logging result (for instance --report_to none)." + ) + return False + + if _is_comet_installed is False: + return False + + if _is_comet_recent_enough is False: + logger.warning( + "comet_ml version %s is installed, but version %s or higher is required. " + "Please update comet_ml to the latest version to enable Comet logging with pip install 'comet-ml>=%s'.", + _comet_version, + _MIN_COMET_VERSION, + _MIN_COMET_VERSION, + ) + return False + + if _is_comet_configured is False: + logger.warning( + "comet_ml is installed but the Comet API Key is not configured. " + "Please set the `COMET_API_KEY` environment variable to enable Comet logging. " + "Check out the documentation for other ways of configuring it: " + "https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#set-the-api-key" + ) + return False + + return True + + +def is_tensorboard_available(): + return importlib.util.find_spec("tensorboard") is not None or importlib.util.find_spec("tensorboardX") is not None + + +def is_optuna_available(): + return importlib.util.find_spec("optuna") is not None + + +def is_ray_available(): + return importlib.util.find_spec("ray") is not None + + +def is_ray_tune_available(): + if not is_ray_available(): + return False + return importlib.util.find_spec("ray.tune") is not None + + +def is_sigopt_available(): + return importlib.util.find_spec("sigopt") is not None + + +def is_azureml_available(): + if importlib.util.find_spec("azureml") is None: + return False + if importlib.util.find_spec("azureml.core") is None: + return False + return importlib.util.find_spec("azureml.core.run") is not None + + +def is_mlflow_available(): + if os.getenv("DISABLE_MLFLOW_INTEGRATION", "FALSE").upper() == "TRUE": + return False + return importlib.util.find_spec("mlflow") is not None + + +def is_dagshub_available(): + return None not in [importlib.util.find_spec("dagshub"), importlib.util.find_spec("mlflow")] + + +def is_neptune_available(): + return _has_neptune + + +def is_codecarbon_available(): + return importlib.util.find_spec("codecarbon") is not None + + +def is_flytekit_available(): + return importlib.util.find_spec("flytekit") is not None + + +def is_flyte_deck_standard_available(): + if not is_flytekit_available(): + return False + return importlib.util.find_spec("flytekitplugins.deck") is not None + + +def is_dvclive_available(): + return importlib.util.find_spec("dvclive") is not None + + +def is_swanlab_available(): + return importlib.util.find_spec("swanlab") is not None + + +def hp_params(trial): + if is_optuna_available(): + import optuna + + if isinstance(trial, optuna.trial.BaseTrial): + return trial.params + if is_ray_tune_available(): + if isinstance(trial, dict): + return trial + + if is_sigopt_available(): + if isinstance(trial, dict): + return trial + + if is_wandb_available(): + if isinstance(trial, dict): + return trial + + raise RuntimeError(f"Unknown type for trial {trial.__class__}") + + +def run_hp_search_optuna(trainer, n_trials: int, direction: str, **kwargs) -> BestRun: + import optuna + from accelerate.utils.memory import release_memory + + if trainer.args.process_index == 0: + + def _objective(trial: optuna.Trial, checkpoint_dir=None): + checkpoint = None + if checkpoint_dir: + for subdir in os.listdir(checkpoint_dir): + if subdir.startswith(PREFIX_CHECKPOINT_DIR): + checkpoint = os.path.join(checkpoint_dir, subdir) + trainer.objective = None + if trainer.args.world_size > 1: + if trainer.args.parallel_mode != ParallelMode.DISTRIBUTED: + raise RuntimeError("only support DDP optuna HPO for ParallelMode.DISTRIBUTED currently.") + trainer.hp_space(trial) + fixed_trial = optuna.trial.FixedTrial(trial.params, trial.number) + trial_main_rank_list = [fixed_trial] + torch.distributed.broadcast_object_list(trial_main_rank_list, src=0) + trainer.train(resume_from_checkpoint=checkpoint, trial=trial) + else: + trainer.train(resume_from_checkpoint=checkpoint, trial=trial) + # If there hasn't been any evaluation during the training loop. + if getattr(trainer, "objective", None) is None: + metrics = trainer.evaluate() + trainer.objective = trainer.compute_objective(metrics) + + # Free GPU memory + trainer.model_wrapped, trainer.model = release_memory(trainer.model_wrapped, trainer.model) + trainer.accelerator.clear() + + return trainer.objective + + timeout = kwargs.pop("timeout", None) + n_jobs = kwargs.pop("n_jobs", 1) + gc_after_trial = kwargs.pop("gc_after_trial", False) + directions = direction if isinstance(direction, list) else None + direction = None if directions is not None else direction + study = optuna.create_study(direction=direction, directions=directions, **kwargs) + study.optimize(_objective, n_trials=n_trials, timeout=timeout, n_jobs=n_jobs, gc_after_trial=gc_after_trial) + if not study._is_multi_objective(): + best_trial = study.best_trial + return BestRun(str(best_trial.number), best_trial.value, best_trial.params) + else: + best_trials = study.best_trials + return [BestRun(str(best.number), best.values, best.params) for best in best_trials] + else: + for i in range(n_trials): + trainer.objective = None + trial_main_rank_list = [None] + if trainer.args.parallel_mode != ParallelMode.DISTRIBUTED: + raise RuntimeError("only support DDP optuna HPO for ParallelMode.DISTRIBUTED currently.") + torch.distributed.broadcast_object_list(trial_main_rank_list, src=0) + trainer.train(resume_from_checkpoint=None, trial=trial_main_rank_list[0]) + # If there hasn't been any evaluation during the training loop. + if getattr(trainer, "objective", None) is None: + metrics = trainer.evaluate() + trainer.objective = trainer.compute_objective(metrics) + return None + + +def run_hp_search_ray(trainer, n_trials: int, direction: str, **kwargs) -> BestRun: + import ray + import ray.train + + def _objective(trial: dict, local_trainer): + try: + from transformers.utils.notebook import NotebookProgressCallback + + if local_trainer.pop_callback(NotebookProgressCallback): + local_trainer.add_callback(ProgressCallback) + except ModuleNotFoundError: + pass + + local_trainer.objective = None + + checkpoint = ray.train.get_checkpoint() + if checkpoint: + # Upon trial resume, the local_trainer's objective gets reset to None. + # If `local_trainer.train` is a noop (training has already reached + # the target number of epochs/steps), then this would + # trigger an unnecessary extra checkpoint at the end of training. + # -> Set the objective to a dummy value upon resume as a workaround. + local_trainer.objective = "objective" + + with checkpoint.as_directory() as checkpoint_dir: + checkpoint_path = next(Path(checkpoint_dir).glob(f"{PREFIX_CHECKPOINT_DIR}*")).as_posix() + local_trainer.train(resume_from_checkpoint=checkpoint_path, trial=trial) + else: + local_trainer.train(trial=trial) + + # If there hasn't been any evaluation during the training loop. + if getattr(local_trainer, "objective", None) is None: + metrics = local_trainer.evaluate() + local_trainer.objective = local_trainer.compute_objective(metrics) + + metrics.update({"objective": local_trainer.objective, "done": True}) + + with tempfile.TemporaryDirectory() as temp_checkpoint_dir: + local_trainer._tune_save_checkpoint(checkpoint_dir=temp_checkpoint_dir) + checkpoint = ray.train.Checkpoint.from_directory(temp_checkpoint_dir) + ray.train.report(metrics, checkpoint=checkpoint) + + if not trainer._memory_tracker.skip_memory_metrics: + from ..trainer_utils import TrainerMemoryTracker + + logger.warning( + "Memory tracking for your Trainer is currently " + "enabled. Automatically disabling the memory tracker " + "since the memory tracker is not serializable." + ) + trainer._memory_tracker = TrainerMemoryTracker(skip_memory_metrics=True) + + # The model and TensorBoard writer do not pickle so we have to remove them (if they exists) + # while doing the ray hp search. + _tb_writer = trainer.pop_callback(TensorBoardCallback) + trainer.model = None + + # Setup default `resources_per_trial`. + if "resources_per_trial" not in kwargs: + # Default to 1 CPU and 1 GPU (if applicable) per trial. + kwargs["resources_per_trial"] = {"cpu": 1} + if trainer.args.n_gpu > 0: + kwargs["resources_per_trial"]["gpu"] = 1 + resource_msg = "1 CPU" + (" and 1 GPU" if trainer.args.n_gpu > 0 else "") + logger.info( + "No `resources_per_trial` arg was passed into " + "`hyperparameter_search`. Setting it to a default value " + f"of {resource_msg} for each trial." + ) + # Make sure each trainer only uses GPUs that were allocated per trial. + gpus_per_trial = kwargs["resources_per_trial"].get("gpu", 0) + trainer.args._n_gpu = gpus_per_trial + + # Setup default `progress_reporter`. + if "progress_reporter" not in kwargs: + from ray.tune import CLIReporter + + kwargs["progress_reporter"] = CLIReporter(metric_columns=["objective"]) + + if "scheduler" in kwargs: + from ray.tune.schedulers import ASHAScheduler, HyperBandForBOHB, MedianStoppingRule, PopulationBasedTraining + + # Check for `do_eval` and `eval_during_training` for schedulers that require intermediate reporting. + if isinstance( + kwargs["scheduler"], (ASHAScheduler, MedianStoppingRule, HyperBandForBOHB, PopulationBasedTraining) + ) and (not trainer.args.do_eval or trainer.args.eval_strategy == IntervalStrategy.NO): + raise RuntimeError( + "You are using {cls} as a scheduler but you haven't enabled evaluation during training. " + "This means your trials will not report intermediate results to Ray Tune, and " + "can thus not be stopped early or used to exploit other trials parameters. " + "If this is what you want, do not use {cls}. If you would like to use {cls}, " + "make sure you pass `do_eval=True` and `eval_strategy='steps'` in the " + "Trainer `args`.".format(cls=type(kwargs["scheduler"]).__name__) + ) + + trainable = ray.tune.with_parameters(_objective, local_trainer=trainer) + + @functools.wraps(trainable) + def dynamic_modules_import_trainable(*args, **kwargs): + """ + Wrapper around `tune.with_parameters` to ensure datasets_modules are loaded on each Actor. + + Without this, an ImportError will be thrown. See https://github.com/huggingface/transformers/issues/11565. + + Assumes that `_objective`, defined above, is a function. + """ + if is_datasets_available(): + import datasets.load + + dynamic_modules_path = os.path.join(datasets.load.init_dynamic_modules(), "__init__.py") + # load dynamic_modules from path + spec = importlib.util.spec_from_file_location("datasets_modules", dynamic_modules_path) + datasets_modules = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = datasets_modules + spec.loader.exec_module(datasets_modules) + return trainable(*args, **kwargs) + + # special attr set by tune.with_parameters + if hasattr(trainable, "__mixins__"): + dynamic_modules_import_trainable.__mixins__ = trainable.__mixins__ + + analysis = ray.tune.run( + dynamic_modules_import_trainable, + config=trainer.hp_space(None), + num_samples=n_trials, + **kwargs, + ) + best_trial = analysis.get_best_trial(metric="objective", mode=direction[:3], scope=trainer.args.ray_scope) + best_run = BestRun(best_trial.trial_id, best_trial.last_result["objective"], best_trial.config, analysis) + if _tb_writer is not None: + trainer.add_callback(_tb_writer) + return best_run + + +def run_hp_search_sigopt(trainer, n_trials: int, direction: str, **kwargs) -> BestRun: + import sigopt + + if trainer.args.process_index == 0: + if importlib.metadata.version("sigopt") >= "8.0.0": + sigopt.set_project("huggingface") + + experiment = sigopt.create_experiment( + name="huggingface-tune", + type="offline", + parameters=trainer.hp_space(None), + metrics=[{"name": "objective", "objective": direction, "strategy": "optimize"}], + parallel_bandwidth=1, + budget=n_trials, + ) + + logger.info(f"created experiment: https://app.sigopt.com/experiment/{experiment.id}") + + for run in experiment.loop(): + with run: + trainer.objective = None + if trainer.args.world_size > 1: + if trainer.args.parallel_mode != ParallelMode.DISTRIBUTED: + raise RuntimeError("only support DDP Sigopt HPO for ParallelMode.DISTRIBUTED currently.") + trainer._hp_search_setup(run.run) + torch.distributed.broadcast_object_list(pickle.dumps(trainer.args), src=0) + trainer.train(resume_from_checkpoint=None) + else: + trainer.train(resume_from_checkpoint=None, trial=run.run) + # If there hasn't been any evaluation during the training loop. + if getattr(trainer, "objective", None) is None: + metrics = trainer.evaluate() + trainer.objective = trainer.compute_objective(metrics) + run.log_metric("objective", trainer.objective) + + best = list(experiment.get_best_runs())[0] + best_run = BestRun(best.id, best.values["objective"].value, best.assignments) + else: + from sigopt import Connection + + conn = Connection() + proxies = kwargs.pop("proxies", None) + if proxies is not None: + conn.set_proxies(proxies) + + experiment = conn.experiments().create( + name="huggingface-tune", + parameters=trainer.hp_space(None), + metrics=[{"name": "objective", "objective": direction, "strategy": "optimize"}], + parallel_bandwidth=1, + observation_budget=n_trials, + project="huggingface", + ) + logger.info(f"created experiment: https://app.sigopt.com/experiment/{experiment.id}") + + while experiment.progress.observation_count < experiment.observation_budget: + suggestion = conn.experiments(experiment.id).suggestions().create() + trainer.objective = None + if trainer.args.world_size > 1: + if trainer.args.parallel_mode != ParallelMode.DISTRIBUTED: + raise RuntimeError("only support DDP Sigopt HPO for ParallelMode.DISTRIBUTED currently.") + trainer._hp_search_setup(suggestion) + torch.distributed.broadcast_object_list(pickle.dumps(trainer.args), src=0) + trainer.train(resume_from_checkpoint=None) + else: + trainer.train(resume_from_checkpoint=None, trial=suggestion) + # If there hasn't been any evaluation during the training loop. + if getattr(trainer, "objective", None) is None: + metrics = trainer.evaluate() + trainer.objective = trainer.compute_objective(metrics) + + values = [{"name": "objective", "value": trainer.objective}] + obs = conn.experiments(experiment.id).observations().create(suggestion=suggestion.id, values=values) + logger.info(f"[suggestion_id, observation_id]: [{suggestion.id}, {obs.id}]") + experiment = conn.experiments(experiment.id).fetch() + + best = list(conn.experiments(experiment.id).best_assignments().fetch().iterate_pages())[0] + best_run = BestRun(best.id, best.value, best.assignments) + return best_run + else: + for i in range(n_trials): + trainer.objective = None + args_main_rank = list(pickle.dumps(trainer.args)) + if trainer.args.parallel_mode != ParallelMode.DISTRIBUTED: + raise RuntimeError("only support DDP Sigopt HPO for ParallelMode.DISTRIBUTED currently.") + torch.distributed.broadcast_object_list(args_main_rank, src=0) + args = pickle.loads(bytes(args_main_rank)) + for key, value in asdict(args).items(): + if key != "local_rank": + setattr(trainer.args, key, value) + trainer.train(resume_from_checkpoint=None) + # If there hasn't been any evaluation during the training loop. + if getattr(trainer, "objective", None) is None: + metrics = trainer.evaluate() + trainer.objective = trainer.compute_objective(metrics) + return None + + +def run_hp_search_wandb(trainer, n_trials: int, direction: str, **kwargs) -> BestRun: + from ..integrations import is_wandb_available + + if not is_wandb_available(): + raise ImportError("This function needs wandb installed: `pip install wandb`") + import wandb + + # add WandbCallback if not already added in trainer callbacks + reporting_to_wandb = False + for callback in trainer.callback_handler.callbacks: + if isinstance(callback, WandbCallback): + reporting_to_wandb = True + break + if not reporting_to_wandb: + trainer.add_callback(WandbCallback()) + trainer.args.report_to = ["wandb"] + best_trial = {"run_id": None, "objective": None, "hyperparameters": None} + sweep_id = kwargs.pop("sweep_id", None) + project = kwargs.pop("project", None) + name = kwargs.pop("name", None) + entity = kwargs.pop("entity", None) + metric = kwargs.pop("metric", "eval/loss") + + sweep_config = trainer.hp_space(None) + sweep_config["metric"]["goal"] = direction + sweep_config["metric"]["name"] = metric + if name: + sweep_config["name"] = name + + def _objective(): + run = wandb.run if wandb.run else wandb.init() + trainer.state.trial_name = run.name + run.config.update({"assignments": {}, "metric": metric}) + config = wandb.config + + trainer.objective = None + + trainer.train(resume_from_checkpoint=None, trial=vars(config)["_items"]) + # If there hasn't been any evaluation during the training loop. + if getattr(trainer, "objective", None) is None: + metrics = trainer.evaluate() + trainer.objective = trainer.compute_objective(metrics) + format_metrics = rewrite_logs(metrics) + if metric not in format_metrics: + logger.warning( + f"Provided metric {metric} not found. This might result in unexpected sweeps charts. The available" + f" metrics are {format_metrics.keys()}" + ) + best_score = False + if best_trial["run_id"] is not None: + if direction == "minimize": + best_score = trainer.objective < best_trial["objective"] + elif direction == "maximize": + best_score = trainer.objective > best_trial["objective"] + + if best_score or best_trial["run_id"] is None: + best_trial["run_id"] = run.id + best_trial["objective"] = trainer.objective + best_trial["hyperparameters"] = dict(config) + + return trainer.objective + + if not sweep_id: + sweep_id = wandb.sweep(sweep_config, project=project, entity=entity) + else: + import wandb.env + + if entity: + wandb.env.set_entity(entity) + wandb.env.set_project(project) + + logger.info(f"wandb sweep id - {sweep_id}") + wandb.agent(sweep_id, function=_objective, count=n_trials) + + return BestRun(best_trial["run_id"], best_trial["objective"], best_trial["hyperparameters"], sweep_id) + + +def get_available_reporting_integrations(): + integrations = [] + if is_azureml_available() and not is_mlflow_available(): + integrations.append("azure_ml") + if is_comet_available(): + integrations.append("comet_ml") + if is_dagshub_available(): + integrations.append("dagshub") + if is_dvclive_available(): + integrations.append("dvclive") + if is_mlflow_available(): + integrations.append("mlflow") + if is_neptune_available(): + integrations.append("neptune") + if is_tensorboard_available(): + integrations.append("tensorboard") + if is_wandb_available(): + integrations.append("wandb") + if is_codecarbon_available(): + integrations.append("codecarbon") + if is_clearml_available(): + integrations.append("clearml") + if is_swanlab_available(): + integrations.append("swanlab") + return integrations + + +def rewrite_logs(d): + new_d = {} + eval_prefix = "eval_" + eval_prefix_len = len(eval_prefix) + test_prefix = "test_" + test_prefix_len = len(test_prefix) + for k, v in d.items(): + if k.startswith(eval_prefix): + new_d["eval/" + k[eval_prefix_len:]] = v + elif k.startswith(test_prefix): + new_d["test/" + k[test_prefix_len:]] = v + else: + new_d["train/" + k] = v + return new_d + + +class TensorBoardCallback(TrainerCallback): + """ + A [`TrainerCallback`] that sends the logs to [TensorBoard](https://www.tensorflow.org/tensorboard). + + Args: + tb_writer (`SummaryWriter`, *optional*): + The writer to use. Will instantiate one if not set. + """ + + def __init__(self, tb_writer=None): + has_tensorboard = is_tensorboard_available() + if not has_tensorboard: + raise RuntimeError( + "TensorBoardCallback requires tensorboard to be installed. Either update your PyTorch version or" + " install tensorboardX." + ) + if has_tensorboard: + try: + from torch.utils.tensorboard import SummaryWriter # noqa: F401 + + self._SummaryWriter = SummaryWriter + except ImportError: + try: + from tensorboardX import SummaryWriter + + self._SummaryWriter = SummaryWriter + except ImportError: + self._SummaryWriter = None + else: + self._SummaryWriter = None + self.tb_writer = tb_writer + + def _init_summary_writer(self, args, log_dir=None): + log_dir = log_dir or args.logging_dir + if self._SummaryWriter is not None: + self.tb_writer = self._SummaryWriter(log_dir=log_dir) + + def on_train_begin(self, args, state, control, **kwargs): + if not state.is_world_process_zero: + return + + log_dir = None + + if state.is_hyper_param_search: + trial_name = state.trial_name + if trial_name is not None: + log_dir = os.path.join(args.logging_dir, trial_name) + + if self.tb_writer is None: + self._init_summary_writer(args, log_dir) + + if self.tb_writer is not None: + self.tb_writer.add_text("args", args.to_json_string()) + if "model" in kwargs: + model = kwargs["model"] + if hasattr(model, "config") and model.config is not None: + model_config_json = model.config.to_json_string() + self.tb_writer.add_text("model_config", model_config_json) + + def on_log(self, args, state, control, logs=None, **kwargs): + if not state.is_world_process_zero: + return + + if self.tb_writer is None: + self._init_summary_writer(args) + + if self.tb_writer is not None: + logs = rewrite_logs(logs) + for k, v in logs.items(): + if isinstance(v, (int, float)): + self.tb_writer.add_scalar(k, v, state.global_step) + elif isinstance(v, str): + self.tb_writer.add_text(k, v, state.global_step) + else: + logger.warning( + "Trainer is attempting to log a value of " + f'"{v}" of type {type(v)} for key "{k}" as a scalar. ' + "This invocation of Tensorboard's writer.add_scalar() " + "is incorrect so we dropped this attribute." + ) + self.tb_writer.flush() + + def on_train_end(self, args, state, control, **kwargs): + if self.tb_writer: + self.tb_writer.close() + self.tb_writer = None + + +def save_model_architecture_to_file(model: Any, output_dir: str): + with open(f"{output_dir}/model_architecture.txt", "w+") as f: + if isinstance(model, PreTrainedModel): + print(model, file=f) + elif is_tf_available() and isinstance(model, TFPreTrainedModel): + + def print_to_file(s): + print(s, file=f) + + model.summary(print_fn=print_to_file) + elif is_torch_available() and ( + isinstance(model, (torch.nn.Module, PushToHubMixin)) and hasattr(model, "base_model") + ): + print(model, file=f) + + +class WandbLogModel(str, Enum): + """Enum of possible log model values in W&B.""" + + CHECKPOINT = "checkpoint" + END = "end" + FALSE = "false" + + @property + def is_enabled(self) -> bool: + """Check if the value corresponds to a state where the `WANDB_LOG_MODEL` setting is enabled.""" + return self in (WandbLogModel.CHECKPOINT, WandbLogModel.END) + + @classmethod + def _missing_(cls, value: Any) -> "WandbLogModel": + if not isinstance(value, str): + raise ValueError(f"Expecting to have a string `WANDB_LOG_MODEL` setting, but got {type(value)}") + if value.upper() in ENV_VARS_TRUE_VALUES: + raise DeprecationWarning( + f"Setting `WANDB_LOG_MODEL` as {os.getenv('WANDB_LOG_MODEL')} is deprecated and will be removed in " + "version 5 of transformers. Use one of `'end'` or `'checkpoint'` instead." + ) + logger.info(f"Setting `WANDB_LOG_MODEL` from {os.getenv('WANDB_LOG_MODEL')} to `end` instead") + return WandbLogModel.END + logger.warning( + f"Received unrecognized `WANDB_LOG_MODEL` setting value={value}; so disabling `WANDB_LOG_MODEL`" + ) + return WandbLogModel.FALSE + + +class WandbCallback(TrainerCallback): + """ + A [`TrainerCallback`] that logs metrics, media, model checkpoints to [Weight and Biases](https://www.wandb.com/). + """ + + def __init__(self): + has_wandb = is_wandb_available() + if not has_wandb: + raise RuntimeError("WandbCallback requires wandb to be installed. Run `pip install wandb`.") + if has_wandb: + import wandb + + self._wandb = wandb + self._initialized = False + self._log_model = WandbLogModel(os.getenv("WANDB_LOG_MODEL", "false")) + + def setup(self, args, state, model, **kwargs): + """ + Setup the optional Weights & Biases (*wandb*) integration. + + One can subclass and override this method to customize the setup if needed. Find more information + [here](https://docs.wandb.ai/guides/integrations/huggingface). You can also override the following environment + variables: + + Environment: + - **WANDB_LOG_MODEL** (`str`, *optional*, defaults to `"false"`): + Whether to log model and checkpoints during training. Can be `"end"`, `"checkpoint"` or `"false"`. If set + to `"end"`, the model will be uploaded at the end of training. If set to `"checkpoint"`, the checkpoint + will be uploaded every `args.save_steps` . If set to `"false"`, the model will not be uploaded. Use along + with [`~transformers.TrainingArguments.load_best_model_at_end`] to upload best model. + + + + Setting `WANDB_LOG_MODEL` as `bool` will be deprecated in version 5 of 🤗 Transformers. + + + - **WANDB_WATCH** (`str`, *optional* defaults to `"false"`): + Can be `"gradients"`, `"all"`, `"parameters"`, or `"false"`. Set to `"all"` to log gradients and + parameters. + - **WANDB_PROJECT** (`str`, *optional*, defaults to `"huggingface"`): + Set this to a custom string to store results in a different project. + - **WANDB_DISABLED** (`bool`, *optional*, defaults to `False`): + Whether to disable wandb entirely. Set `WANDB_DISABLED=true` to disable. + """ + if self._wandb is None: + return + self._initialized = True + + # prepare to handle potential configuration issues during setup + from wandb.sdk.lib.config_util import ConfigError as WandbConfigError + + if state.is_world_process_zero: + logger.info( + 'Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"' + ) + combined_dict = {**args.to_dict()} + + if hasattr(model, "config") and model.config is not None: + model_config = model.config if isinstance(model.config, dict) else model.config.to_dict() + combined_dict = {**model_config, **combined_dict} + if hasattr(model, "peft_config") and model.peft_config is not None: + peft_config = model.peft_config + combined_dict = {**{"peft_config": peft_config}, **combined_dict} + trial_name = state.trial_name + init_args = {} + if trial_name is not None: + init_args["name"] = trial_name + init_args["group"] = args.run_name + elif args.run_name is not None: + init_args["name"] = args.run_name + if args.run_name == args.output_dir: + self._wandb.termwarn( + "The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was " + "not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.", + repeat=False, + ) + + if self._wandb.run is None: + self._wandb.init( + project=os.getenv("WANDB_PROJECT", "huggingface"), + **init_args, + ) + # add config parameters (run may have been created manually) + self._wandb.config.update(combined_dict, allow_val_change=True) + + # define default x-axis (for latest wandb versions) + if getattr(self._wandb, "define_metric", None): + self._wandb.define_metric("train/global_step") + self._wandb.define_metric("*", step_metric="train/global_step", step_sync=True) + + # keep track of model topology and gradients, unsupported on TPU + _watch_model = os.getenv("WANDB_WATCH", "false") + if not is_torch_xla_available() and _watch_model in ("all", "parameters", "gradients"): + self._wandb.watch(model, log=_watch_model, log_freq=max(100, state.logging_steps)) + self._wandb.run._label(code="transformers_trainer") + + # add number of model parameters to wandb config + try: + self._wandb.config["model/num_parameters"] = model.num_parameters() + except AttributeError: + logger.info( + "Could not log the number of model parameters in Weights & Biases due to an AttributeError." + ) + except WandbConfigError: + logger.warning( + "A ConfigError was raised whilst setting the number of model parameters in Weights & Biases config." + ) + + # log the initial model architecture to an artifact + if self._log_model.is_enabled: + with tempfile.TemporaryDirectory() as temp_dir: + model_name = ( + f"model-{self._wandb.run.id}" + if (args.run_name is None or args.run_name == args.output_dir) + else f"model-{self._wandb.run.name}" + ) + model_artifact = self._wandb.Artifact( + name=model_name, + type="model", + metadata={ + "model_config": model.config.to_dict() if hasattr(model, "config") else None, + "num_parameters": self._wandb.config.get("model/num_parameters"), + "initial_model": True, + }, + ) + # add the architecture to a separate text file + save_model_architecture_to_file(model, temp_dir) + + for f in Path(temp_dir).glob("*"): + if f.is_file(): + with model_artifact.new_file(f.name, mode="wb") as fa: + fa.write(f.read_bytes()) + self._wandb.run.log_artifact(model_artifact, aliases=["base_model"]) + + badge_markdown = ( + f'[Visualize in Weights & Biases]({self._wandb.run.get_url()})' + ) + + modelcard.AUTOGENERATED_TRAINER_COMMENT += f"\n{badge_markdown}" + + def on_train_begin(self, args, state, control, model=None, **kwargs): + if self._wandb is None: + return + hp_search = state.is_hyper_param_search + if hp_search: + self._wandb.finish() + self._initialized = False + args.run_name = None + if not self._initialized: + self.setup(args, state, model, **kwargs) + + def on_train_end(self, args, state, control, model=None, processing_class=None, **kwargs): + if self._wandb is None: + return + if self._log_model.is_enabled and self._initialized and state.is_world_process_zero: + from ..trainer import Trainer + + fake_trainer = Trainer(args=args, model=model, processing_class=processing_class, eval_dataset=["fake"]) + with tempfile.TemporaryDirectory() as temp_dir: + fake_trainer.save_model(temp_dir) + metadata = ( + { + k: v + for k, v in dict(self._wandb.summary).items() + if isinstance(v, numbers.Number) and not k.startswith("_") + } + if not args.load_best_model_at_end + else { + f"eval/{args.metric_for_best_model}": state.best_metric, + "train/total_floss": state.total_flos, + "model/num_parameters": self._wandb.config.get("model/num_parameters"), + } + ) + metadata["final_model"] = True + logger.info("Logging model artifacts. ...") + model_name = ( + f"model-{self._wandb.run.id}" + if (args.run_name is None or args.run_name == args.output_dir) + else f"model-{self._wandb.run.name}" + ) + # add the model architecture to a separate text file + save_model_architecture_to_file(model, temp_dir) + + artifact = self._wandb.Artifact(name=model_name, type="model", metadata=metadata) + for f in Path(temp_dir).glob("*"): + if f.is_file(): + with artifact.new_file(f.name, mode="wb") as fa: + fa.write(f.read_bytes()) + self._wandb.run.log_artifact(artifact, aliases=["final_model"]) + + def on_log(self, args, state, control, model=None, logs=None, **kwargs): + single_value_scalars = [ + "train_runtime", + "train_samples_per_second", + "train_steps_per_second", + "train_loss", + "total_flos", + ] + + if self._wandb is None: + return + if not self._initialized: + self.setup(args, state, model) + if state.is_world_process_zero: + for k, v in logs.items(): + if k in single_value_scalars: + self._wandb.run.summary[k] = v + non_scalar_logs = {k: v for k, v in logs.items() if k not in single_value_scalars} + non_scalar_logs = rewrite_logs(non_scalar_logs) + self._wandb.log({**non_scalar_logs, "train/global_step": state.global_step}) + + def on_save(self, args, state, control, **kwargs): + if self._log_model == WandbLogModel.CHECKPOINT and self._initialized and state.is_world_process_zero: + checkpoint_metadata = { + k: v + for k, v in dict(self._wandb.summary).items() + if isinstance(v, numbers.Number) and not k.startswith("_") + } + checkpoint_metadata["model/num_parameters"] = self._wandb.config.get("model/num_parameters") + + ckpt_dir = f"checkpoint-{state.global_step}" + artifact_path = os.path.join(args.output_dir, ckpt_dir) + logger.info(f"Logging checkpoint artifacts in {ckpt_dir}. ...") + checkpoint_name = ( + f"model-{self._wandb.run.id}" + if (args.run_name is None or args.run_name == args.output_dir) + else f"model-{self._wandb.run.name}" + ) + artifact = self._wandb.Artifact(name=checkpoint_name, type="model", metadata=checkpoint_metadata) + artifact.add_dir(artifact_path) + self._wandb.log_artifact( + artifact, aliases=[f"epoch_{round(state.epoch, 2)}", f"checkpoint_global_step_{state.global_step}"] + ) + + def on_predict(self, args, state, control, metrics, **kwargs): + if self._wandb is None: + return + if not self._initialized: + self.setup(args, state, **kwargs) + if state.is_world_process_zero: + metrics = rewrite_logs(metrics) + self._wandb.log(metrics) + + +class CometCallback(TrainerCallback): + """ + A [`TrainerCallback`] that sends the logs to [Comet ML](https://www.comet.com/site/). + """ + + def __init__(self): + if _is_comet_installed is False or _is_comet_recent_enough is False: + raise RuntimeError( + f"CometCallback requires comet-ml>={_MIN_COMET_VERSION} to be installed. Run `pip install comet-ml>={_MIN_COMET_VERSION}`." + ) + self._initialized = False + self._log_assets = False + self._experiment = None + + def setup(self, args, state, model): + """ + Setup the optional Comet integration. + + Environment: + - **COMET_MODE** (`str`, *optional*, default to `get_or_create`): + Control whether to create and log to a new Comet experiment or append to an existing experiment. + It accepts the following values: + * `get_or_create`: Decides automatically depending if + `COMET_EXPERIMENT_KEY` is set and whether an Experiment + with that key already exists or not. + * `create`: Always create a new Comet Experiment. + * `get`: Always try to append to an Existing Comet Experiment. + Requires `COMET_EXPERIMENT_KEY` to be set. + * `ONLINE`: **deprecated**, used to create an online + Experiment. Use `COMET_START_ONLINE=1` instead. + * `OFFLINE`: **deprecated**, used to created an offline + Experiment. Use `COMET_START_ONLINE=0` instead. + * `DISABLED`: **deprecated**, used to disable Comet logging. + Use the `--report_to` flag to control the integrations used + for logging result instead. + - **COMET_PROJECT_NAME** (`str`, *optional*): + Comet project name for experiments. + - **COMET_LOG_ASSETS** (`str`, *optional*, defaults to `TRUE`): + Whether or not to log training assets (tf event logs, checkpoints, etc), to Comet. Can be `TRUE`, or + `FALSE`. + + For a number of configurable items in the environment, see + [here](https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options). + """ + self._initialized = True + log_assets = os.getenv("COMET_LOG_ASSETS", "FALSE").upper() + if log_assets in {"TRUE", "1"}: + self._log_assets = True + if state.is_world_process_zero: + comet_old_mode = os.getenv("COMET_MODE") + + mode = None + online = None + + if comet_old_mode is not None: + comet_old_mode = comet_old_mode.lower() + + if comet_old_mode == "online": + online = True + elif comet_old_mode == "offline": + online = False + elif comet_old_mode in ("get", "get_or_create", "create"): + mode = comet_old_mode + elif comet_old_mode: + logger.warning("Invalid COMET_MODE env value %r, Comet logging is disabled", comet_old_mode) + return + + # For HPO, we always create a new experiment for each trial + if state.is_hyper_param_search: + if mode is not None: + logger.warning( + "Hyperparameter Search is enabled, forcing the creation of new experimetns, COMET_MODE value %r is ignored", + comet_old_mode, + ) + mode = "create" + + import comet_ml + + # Do not use the default run_name as the experiment name + if args.run_name is not None and args.run_name != args.output_dir: + experiment_config = comet_ml.ExperimentConfig(name=args.run_name) + else: + experiment_config = comet_ml.ExperimentConfig() + + self._experiment = comet_ml.start(online=online, mode=mode, experiment_config=experiment_config) + self._experiment.__internal_api__set_model_graph__(model, framework="transformers") + + params = {"args": args.to_dict()} + + if hasattr(model, "config") and model.config is not None: + model_config = model.config.to_dict() + params["config"] = model_config + if hasattr(model, "peft_config") and model.peft_config is not None: + peft_config = model.peft_config + params["peft_config"] = peft_config + + self._experiment.__internal_api__log_parameters__( + params, framework="transformers", source="manual", flatten_nested=True + ) + + if state.is_hyper_param_search: + optimization_id = getattr(state, "trial_name", None) + optimization_params = getattr(state, "trial_params", None) + + self._experiment.log_optimization(optimization_id=optimization_id, parameters=optimization_params) + + def on_train_begin(self, args, state, control, model=None, **kwargs): + if not self._initialized: + self.setup(args, state, model) + + def on_log(self, args, state, control, model=None, logs=None, **kwargs): + if not self._initialized: + self.setup(args, state, model) + if state.is_world_process_zero: + if self._experiment is not None: + rewritten_logs = rewrite_logs(logs) + self._experiment.__internal_api__log_metrics__( + rewritten_logs, step=state.global_step, epoch=state.epoch, framework="transformers" + ) + + def on_train_end(self, args, state, control, **kwargs): + if self._initialized and state.is_world_process_zero: + if self._experiment is not None: + if self._log_assets is True: + logger.info("Logging checkpoints. This may take time.") + self._experiment.log_asset_folder( + args.output_dir, recursive=True, log_file_name=True, step=state.global_step + ) + + # We create one experiment per trial in HPO mode + if state.is_hyper_param_search: + self._experiment.clean() + self._initialized = False + + def on_predict(self, args, state, control, metrics, **kwargs): + if not self._initialized: + self.setup(args, state, model=None) + if state.is_world_process_zero and self._experiment is not None: + rewritten_metrics = rewrite_logs(metrics) + self._experiment.__internal_api__log_metrics__( + rewritten_metrics, step=state.global_step, epoch=state.epoch, framework="transformers" + ) + + +class AzureMLCallback(TrainerCallback): + """ + A [`TrainerCallback`] that sends the logs to [AzureML](https://pypi.org/project/azureml-sdk/). + """ + + def __init__(self, azureml_run=None): + if not is_azureml_available(): + raise RuntimeError("AzureMLCallback requires azureml to be installed. Run `pip install azureml-sdk`.") + self.azureml_run = azureml_run + + def on_init_end(self, args, state, control, **kwargs): + from azureml.core.run import Run + + if self.azureml_run is None and state.is_world_process_zero: + self.azureml_run = Run.get_context() + + def on_log(self, args, state, control, logs=None, **kwargs): + if self.azureml_run and state.is_world_process_zero: + for k, v in logs.items(): + if isinstance(v, (int, float)): + self.azureml_run.log(k, v, description=k) + + +class MLflowCallback(TrainerCallback): + """ + A [`TrainerCallback`] that sends the logs to [MLflow](https://www.mlflow.org/). Can be disabled by setting + environment variable `DISABLE_MLFLOW_INTEGRATION = TRUE`. + """ + + def __init__(self): + if not is_mlflow_available(): + raise RuntimeError("MLflowCallback requires mlflow to be installed. Run `pip install mlflow`.") + import mlflow + + self._MAX_PARAM_VAL_LENGTH = mlflow.utils.validation.MAX_PARAM_VAL_LENGTH + self._MAX_PARAMS_TAGS_PER_BATCH = mlflow.utils.validation.MAX_PARAMS_TAGS_PER_BATCH + + self._initialized = False + self._auto_end_run = False + self._log_artifacts = False + self._ml_flow = mlflow + + def setup(self, args, state, model): + """ + Setup the optional MLflow integration. + + Environment: + - **HF_MLFLOW_LOG_ARTIFACTS** (`str`, *optional*): + Whether to use MLflow `.log_artifact()` facility to log artifacts. This only makes sense if logging to a + remote server, e.g. s3 or GCS. If set to `True` or *1*, will copy each saved checkpoint on each save in + [`TrainingArguments`]'s `output_dir` to the local or remote artifact storage. Using it without a remote + storage will just copy the files to your artifact location. + - **MLFLOW_TRACKING_URI** (`str`, *optional*): + Whether to store runs at a specific path or remote server. Unset by default, which skips setting the + tracking URI entirely. + - **MLFLOW_EXPERIMENT_NAME** (`str`, *optional*, defaults to `None`): + Whether to use an MLflow experiment_name under which to launch the run. Default to `None` which will point + to the `Default` experiment in MLflow. Otherwise, it is a case sensitive name of the experiment to be + activated. If an experiment with this name does not exist, a new experiment with this name is created. + - **MLFLOW_TAGS** (`str`, *optional*): + A string dump of a dictionary of key/value pair to be added to the MLflow run as tags. Example: + `os.environ['MLFLOW_TAGS']='{"release.candidate": "RC1", "release.version": "2.2.0"}'`. + - **MLFLOW_NESTED_RUN** (`str`, *optional*): + Whether to use MLflow nested runs. If set to `True` or *1*, will create a nested run inside the current + run. + - **MLFLOW_RUN_ID** (`str`, *optional*): + Allow to reattach to an existing run which can be useful when resuming training from a checkpoint. When + `MLFLOW_RUN_ID` environment variable is set, `start_run` attempts to resume a run with the specified run ID + and other parameters are ignored. + - **MLFLOW_FLATTEN_PARAMS** (`str`, *optional*, defaults to `False`): + Whether to flatten the parameters dictionary before logging. + - **MLFLOW_MAX_LOG_PARAMS** (`int`, *optional*): + Set the maximum number of parameters to log in the run. + """ + self._log_artifacts = os.getenv("HF_MLFLOW_LOG_ARTIFACTS", "FALSE").upper() in ENV_VARS_TRUE_VALUES + self._nested_run = os.getenv("MLFLOW_NESTED_RUN", "FALSE").upper() in ENV_VARS_TRUE_VALUES + self._tracking_uri = os.getenv("MLFLOW_TRACKING_URI", None) + self._experiment_name = os.getenv("MLFLOW_EXPERIMENT_NAME", None) + self._flatten_params = os.getenv("MLFLOW_FLATTEN_PARAMS", "FALSE").upper() in ENV_VARS_TRUE_VALUES + self._run_id = os.getenv("MLFLOW_RUN_ID", None) + self._max_log_params = os.getenv("MLFLOW_MAX_LOG_PARAMS", None) + + # "synchronous" flag is only available with mlflow version >= 2.8.0 + # https://github.com/mlflow/mlflow/pull/9705 + # https://github.com/mlflow/mlflow/releases/tag/v2.8.0 + self._async_log = packaging.version.parse(self._ml_flow.__version__) >= packaging.version.parse("2.8.0") + + logger.debug( + f"MLflow experiment_name={self._experiment_name}, run_name={args.run_name}, nested={self._nested_run}," + f" tracking_uri={self._tracking_uri}" + ) + if state.is_world_process_zero: + if not self._ml_flow.is_tracking_uri_set(): + if self._tracking_uri: + self._ml_flow.set_tracking_uri(self._tracking_uri) + logger.debug(f"MLflow tracking URI is set to {self._tracking_uri}") + else: + logger.debug( + "Environment variable `MLFLOW_TRACKING_URI` is not provided and therefore will not be" + " explicitly set." + ) + else: + logger.debug(f"MLflow tracking URI is set to {self._ml_flow.get_tracking_uri()}") + + if self._ml_flow.active_run() is None or self._nested_run or self._run_id: + if self._experiment_name: + # Use of set_experiment() ensure that Experiment is created if not exists + self._ml_flow.set_experiment(self._experiment_name) + self._ml_flow.start_run(run_name=args.run_name, nested=self._nested_run) + logger.debug(f"MLflow run started with run_id={self._ml_flow.active_run().info.run_id}") + self._auto_end_run = True + combined_dict = args.to_dict() + if hasattr(model, "config") and model.config is not None: + model_config = model.config.to_dict() + combined_dict = {**model_config, **combined_dict} + combined_dict = flatten_dict(combined_dict) if self._flatten_params else combined_dict + # remove params that are too long for MLflow + for name, value in list(combined_dict.items()): + # internally, all values are converted to str in MLflow + if len(str(value)) > self._MAX_PARAM_VAL_LENGTH: + logger.warning( + f'Trainer is attempting to log a value of "{value}" for key "{name}" as a parameter. MLflow\'s' + " log_param() only accepts values no longer than 250 characters so we dropped this attribute." + " You can use `MLFLOW_FLATTEN_PARAMS` environment variable to flatten the parameters and" + " avoid this message." + ) + del combined_dict[name] + # MLflow cannot log more than 100 values in one go, so we have to split it + combined_dict_items = list(combined_dict.items()) + if self._max_log_params and self._max_log_params.isdigit(): + max_log_params = int(self._max_log_params) + if max_log_params < len(combined_dict_items): + logger.debug( + f"Reducing the number of parameters to log from {len(combined_dict_items)} to {max_log_params}." + ) + combined_dict_items = combined_dict_items[:max_log_params] + for i in range(0, len(combined_dict_items), self._MAX_PARAMS_TAGS_PER_BATCH): + if self._async_log: + self._ml_flow.log_params( + dict(combined_dict_items[i : i + self._MAX_PARAMS_TAGS_PER_BATCH]), synchronous=False + ) + else: + self._ml_flow.log_params(dict(combined_dict_items[i : i + self._MAX_PARAMS_TAGS_PER_BATCH])) + mlflow_tags = os.getenv("MLFLOW_TAGS", None) + if mlflow_tags: + mlflow_tags = json.loads(mlflow_tags) + self._ml_flow.set_tags(mlflow_tags) + self._initialized = True + + def on_train_begin(self, args, state, control, model=None, **kwargs): + if not self._initialized: + self.setup(args, state, model) + + def on_log(self, args, state, control, logs, model=None, **kwargs): + if not self._initialized: + self.setup(args, state, model) + if state.is_world_process_zero: + metrics = {} + for k, v in logs.items(): + if isinstance(v, (int, float)): + metrics[k] = v + elif isinstance(v, torch.Tensor) and v.numel() == 1: + metrics[k] = v.item() + else: + logger.warning( + f'Trainer is attempting to log a value of "{v}" of type {type(v)} for key "{k}" as a metric. ' + "MLflow's log_metric() only accepts float and int types so we dropped this attribute." + ) + + if self._async_log: + self._ml_flow.log_metrics(metrics=metrics, step=state.global_step, synchronous=False) + else: + self._ml_flow.log_metrics(metrics=metrics, step=state.global_step) + + def on_train_end(self, args, state, control, **kwargs): + if self._initialized and state.is_world_process_zero: + if self._auto_end_run and self._ml_flow.active_run(): + self._ml_flow.end_run() + + def on_save(self, args, state, control, **kwargs): + if self._initialized and state.is_world_process_zero and self._log_artifacts: + ckpt_dir = f"checkpoint-{state.global_step}" + artifact_path = os.path.join(args.output_dir, ckpt_dir) + logger.info(f"Logging checkpoint artifacts in {ckpt_dir}. This may take time.") + self._ml_flow.pyfunc.log_model( + ckpt_dir, + artifacts={"model_path": artifact_path}, + python_model=self._ml_flow.pyfunc.PythonModel(), + ) + + def __del__(self): + # if the previous run is not terminated correctly, the fluent API will + # not let you start a new run before the previous one is killed + if ( + self._auto_end_run + and callable(getattr(self._ml_flow, "active_run", None)) + and self._ml_flow.active_run() is not None + ): + self._ml_flow.end_run() + + +class DagsHubCallback(MLflowCallback): + """ + A [`TrainerCallback`] that logs to [DagsHub](https://dagshub.com/). Extends [`MLflowCallback`] + """ + + def __init__(self): + super().__init__() + if not is_dagshub_available(): + raise ImportError("DagsHubCallback requires dagshub to be installed. Run `pip install dagshub`.") + + from dagshub.upload import Repo + + self.Repo = Repo + + def setup(self, *args, **kwargs): + """ + Setup the DagsHub's Logging integration. + + Environment: + - **HF_DAGSHUB_LOG_ARTIFACTS** (`str`, *optional*): + Whether to save the data and model artifacts for the experiment. Default to `False`. + """ + + self.log_artifacts = os.getenv("HF_DAGSHUB_LOG_ARTIFACTS", "FALSE").upper() in ENV_VARS_TRUE_VALUES + self.name = os.getenv("HF_DAGSHUB_MODEL_NAME") or "main" + self.remote = os.getenv("MLFLOW_TRACKING_URI") + self.repo = self.Repo( + owner=self.remote.split(os.sep)[-2], + name=self.remote.split(os.sep)[-1].split(".")[0], + branch=os.getenv("BRANCH") or "main", + ) + self.path = Path("artifacts") + + if self.remote is None: + raise RuntimeError( + "DagsHubCallback requires the `MLFLOW_TRACKING_URI` environment variable to be set. Did you run" + " `dagshub.init()`?" + ) + + super().setup(*args, **kwargs) + + def on_train_end(self, args, state, control, **kwargs): + if self.log_artifacts: + if getattr(self, "train_dataloader", None): + torch.save(self.train_dataloader.dataset, os.path.join(args.output_dir, "dataset.pt")) + + self.repo.directory(str(self.path)).add_dir(args.output_dir) + + +class NeptuneMissingConfiguration(Exception): + def __init__(self): + super().__init__( + """ + ------ Unsupported ---- We were not able to create new runs. You provided a custom Neptune run to + `NeptuneCallback` with the `run` argument. For the integration to work fully, provide your `api_token` and + `project` by saving them as environment variables or passing them to the callback. + """ + ) + + +class NeptuneCallback(TrainerCallback): + """TrainerCallback that sends the logs to [Neptune](https://app.neptune.ai). + + Args: + api_token (`str`, *optional*): Neptune API token obtained upon registration. + You can leave this argument out if you have saved your token to the `NEPTUNE_API_TOKEN` environment + variable (strongly recommended). See full setup instructions in the + [docs](https://docs.neptune.ai/setup/installation). + project (`str`, *optional*): Name of an existing Neptune project, in the form "workspace-name/project-name". + You can find and copy the name in Neptune from the project settings -> Properties. If None (default), the + value of the `NEPTUNE_PROJECT` environment variable is used. + name (`str`, *optional*): Custom name for the run. + base_namespace (`str`, *optional*, defaults to "finetuning"): In the Neptune run, the root namespace + that will contain all of the metadata logged by the callback. + log_parameters (`bool`, *optional*, defaults to `True`): + If True, logs all Trainer arguments and model parameters provided by the Trainer. + log_checkpoints (`str`, *optional*): If "same", uploads checkpoints whenever they are saved by the Trainer. + If "last", uploads only the most recently saved checkpoint. If "best", uploads the best checkpoint (among + the ones saved by the Trainer). If `None`, does not upload checkpoints. + run (`Run`, *optional*): Pass a Neptune run object if you want to continue logging to an existing run. + Read more about resuming runs in the [docs](https://docs.neptune.ai/logging/to_existing_object). + **neptune_run_kwargs (*optional*): + Additional keyword arguments to be passed directly to the + [`neptune.init_run()`](https://docs.neptune.ai/api/neptune#init_run) function when a new run is created. + + For instructions and examples, see the [Transformers integration + guide](https://docs.neptune.ai/integrations/transformers) in the Neptune documentation. + """ + + integration_version_key = "source_code/integrations/transformers" + model_parameters_key = "model_parameters" + trial_name_key = "trial" + trial_params_key = "trial_params" + trainer_parameters_key = "trainer_parameters" + flat_metrics = {"train/epoch"} + + def __init__( + self, + *, + api_token: Optional[str] = None, + project: Optional[str] = None, + name: Optional[str] = None, + base_namespace: str = "finetuning", + run=None, + log_parameters: bool = True, + log_checkpoints: Optional[str] = None, + **neptune_run_kwargs, + ): + if not is_neptune_available(): + raise ValueError( + "NeptuneCallback requires the Neptune client library to be installed. " + "To install the library, run `pip install neptune`." + ) + + try: + from neptune import Run + from neptune.internal.utils import verify_type + except ImportError: + from neptune.new.internal.utils import verify_type + from neptune.new.metadata_containers.run import Run + + verify_type("api_token", api_token, (str, type(None))) + verify_type("project", project, (str, type(None))) + verify_type("name", name, (str, type(None))) + verify_type("base_namespace", base_namespace, str) + verify_type("run", run, (Run, type(None))) + verify_type("log_parameters", log_parameters, bool) + verify_type("log_checkpoints", log_checkpoints, (str, type(None))) + + self._base_namespace_path = base_namespace + self._log_parameters = log_parameters + self._log_checkpoints = log_checkpoints + self._initial_run: Optional[Run] = run + + self._run = None + self._is_monitoring_run = False + self._run_id = None + self._force_reset_monitoring_run = False + self._init_run_kwargs = {"api_token": api_token, "project": project, "name": name, **neptune_run_kwargs} + + self._volatile_checkpoints_dir = None + self._should_upload_checkpoint = self._log_checkpoints is not None + self._recent_checkpoint_path = None + + if self._log_checkpoints in {"last", "best"}: + self._target_checkpoints_namespace = f"checkpoints/{self._log_checkpoints}" + self._should_clean_recently_uploaded_checkpoint = True + else: + self._target_checkpoints_namespace = "checkpoints" + self._should_clean_recently_uploaded_checkpoint = False + + def _stop_run_if_exists(self): + if self._run: + self._run.stop() + del self._run + self._run = None + + def _initialize_run(self, **additional_neptune_kwargs): + try: + from neptune import init_run + from neptune.exceptions import NeptuneMissingApiTokenException, NeptuneMissingProjectNameException + except ImportError: + from neptune.new import init_run + from neptune.new.exceptions import NeptuneMissingApiTokenException, NeptuneMissingProjectNameException + + self._stop_run_if_exists() + + try: + run_params = additional_neptune_kwargs.copy() + run_params.update(self._init_run_kwargs) + self._run = init_run(**run_params) + self._run_id = self._run["sys/id"].fetch() + except (NeptuneMissingProjectNameException, NeptuneMissingApiTokenException) as e: + raise NeptuneMissingConfiguration() from e + + def _use_initial_run(self): + self._run = self._initial_run + self._is_monitoring_run = True + self._run_id = self._run["sys/id"].fetch() + self._initial_run = None + + def _ensure_run_with_monitoring(self): + if self._initial_run is not None: + self._use_initial_run() + else: + if not self._force_reset_monitoring_run and self._is_monitoring_run: + return + + if self._run and not self._is_monitoring_run and not self._force_reset_monitoring_run: + self._initialize_run(with_id=self._run_id) + self._is_monitoring_run = True + else: + self._initialize_run() + self._force_reset_monitoring_run = False + + def _ensure_at_least_run_without_monitoring(self): + if self._initial_run is not None: + self._use_initial_run() + else: + if not self._run: + self._initialize_run( + with_id=self._run_id, + capture_stdout=False, + capture_stderr=False, + capture_hardware_metrics=False, + capture_traceback=False, + ) + self._is_monitoring_run = False + + @property + def run(self): + if self._run is None: + self._ensure_at_least_run_without_monitoring() + return self._run + + @property + def _metadata_namespace(self): + return self.run[self._base_namespace_path] + + def _log_integration_version(self): + self.run[NeptuneCallback.integration_version_key] = version + + def _log_trainer_parameters(self, args): + self._metadata_namespace[NeptuneCallback.trainer_parameters_key] = args.to_sanitized_dict() + + def _log_model_parameters(self, model): + from neptune.utils import stringify_unsupported + + if model and hasattr(model, "config") and model.config is not None: + self._metadata_namespace[NeptuneCallback.model_parameters_key] = stringify_unsupported( + model.config.to_dict() + ) + + def _log_hyper_param_search_parameters(self, state): + if state and hasattr(state, "trial_name"): + self._metadata_namespace[NeptuneCallback.trial_name_key] = state.trial_name + + if state and hasattr(state, "trial_params") and state.trial_params is not None: + self._metadata_namespace[NeptuneCallback.trial_params_key] = state.trial_params + + def _log_model_checkpoint(self, source_directory: str, checkpoint: str): + target_path = relative_path = os.path.join(source_directory, checkpoint) + + if self._volatile_checkpoints_dir is not None: + consistent_checkpoint_path = os.path.join(self._volatile_checkpoints_dir, checkpoint) + try: + # Remove leading ../ from a relative path. + cpkt_path = relative_path.replace("..", "").lstrip(os.path.sep) + copy_path = os.path.join(consistent_checkpoint_path, cpkt_path) + shutil.copytree(relative_path, copy_path) + target_path = consistent_checkpoint_path + except IOError as e: + logger.warning( + "NeptuneCallback was unable to made a copy of checkpoint due to I/O exception: '{}'. " + "Could fail trying to upload.".format(e) + ) + + self._metadata_namespace[self._target_checkpoints_namespace].upload_files(target_path) + + if self._should_clean_recently_uploaded_checkpoint and self._recent_checkpoint_path is not None: + self._metadata_namespace[self._target_checkpoints_namespace].delete_files(self._recent_checkpoint_path) + + self._recent_checkpoint_path = relative_path + + def on_init_end(self, args, state, control, **kwargs): + self._volatile_checkpoints_dir = None + if self._log_checkpoints and (args.overwrite_output_dir or args.save_total_limit is not None): + self._volatile_checkpoints_dir = tempfile.TemporaryDirectory().name + + if self._log_checkpoints == "best" and not args.load_best_model_at_end: + raise ValueError("To save the best model checkpoint, the load_best_model_at_end argument must be enabled.") + + def on_train_begin(self, args, state, control, model=None, **kwargs): + if not state.is_world_process_zero: + return + + self._ensure_run_with_monitoring() + self._force_reset_monitoring_run = True + + self._log_integration_version() + if self._log_parameters: + self._log_trainer_parameters(args) + self._log_model_parameters(model) + + if state.is_hyper_param_search: + self._log_hyper_param_search_parameters(state) + + def on_train_end(self, args, state, control, **kwargs): + self._stop_run_if_exists() + + def __del__(self): + if self._volatile_checkpoints_dir is not None: + shutil.rmtree(self._volatile_checkpoints_dir, ignore_errors=True) + + self._stop_run_if_exists() + + def on_save(self, args, state, control, **kwargs): + if self._should_upload_checkpoint: + self._log_model_checkpoint(args.output_dir, f"checkpoint-{state.global_step}") + + def on_evaluate(self, args, state, control, metrics=None, **kwargs): + if self._log_checkpoints == "best": + best_metric_name = args.metric_for_best_model + if not best_metric_name.startswith("eval_"): + best_metric_name = f"eval_{best_metric_name}" + + metric_value = metrics.get(best_metric_name) + + operator = np.greater if args.greater_is_better else np.less + + self._should_upload_checkpoint = state.best_metric is None or operator(metric_value, state.best_metric) + + @classmethod + def get_run(cls, trainer): + for callback in trainer.callback_handler.callbacks: + if isinstance(callback, cls): + return callback.run + + raise Exception("The trainer doesn't have a NeptuneCallback configured.") + + def on_log(self, args, state, control, logs: Optional[Dict[str, float]] = None, **kwargs): + if not state.is_world_process_zero: + return + + if logs is not None: + for name, value in rewrite_logs(logs).items(): + if isinstance(value, (int, float)): + if name in NeptuneCallback.flat_metrics: + self._metadata_namespace[name] = value + else: + self._metadata_namespace[name].log(value, step=state.global_step) + + +class CodeCarbonCallback(TrainerCallback): + """ + A [`TrainerCallback`] that tracks the CO2 emission of training. + """ + + def __init__(self): + if not is_codecarbon_available(): + raise RuntimeError( + "CodeCarbonCallback requires `codecarbon` to be installed. Run `pip install codecarbon`." + ) + elif torch.version.hip: + raise RuntimeError( + "CodeCarbonCallback requires `codecarbon` package, which is not compatible with AMD ROCm (https://github.com/mlco2/codecarbon/pull/490). When using the Trainer, please specify the `report_to` argument (https://huggingface.co/docs/transformers/v4.39.3/en/main_classes/trainer#transformers.TrainingArguments.report_to) to disable CodeCarbonCallback." + ) + + import codecarbon + + self._codecarbon = codecarbon + self.tracker = None + + def on_init_end(self, args, state, control, **kwargs): + if self.tracker is None and state.is_local_process_zero: + # CodeCarbon will automatically handle environment variables for configuration + self.tracker = self._codecarbon.EmissionsTracker(output_dir=args.output_dir) + + def on_train_begin(self, args, state, control, model=None, **kwargs): + if self.tracker and state.is_local_process_zero: + self.tracker.start() + + def on_train_end(self, args, state, control, **kwargs): + if self.tracker and state.is_local_process_zero: + self.tracker.stop() + + +class ClearMLCallback(TrainerCallback): + """ + A [`TrainerCallback`] that sends the logs to [ClearML](https://clear.ml/). + + Environment: + - **CLEARML_PROJECT** (`str`, *optional*, defaults to `HuggingFace Transformers`): + ClearML project name. + - **CLEARML_TASK** (`str`, *optional*, defaults to `Trainer`): + ClearML task name. + - **CLEARML_LOG_MODEL** (`bool`, *optional*, defaults to `False`): + Whether to log models as artifacts during training. + """ + + log_suffix = "" + + _hparams_section = "Transformers" + _model_config_section = "Model Configuration" + _ignore_hparams_overrides = "_ignore_hparams_ui_overrides_" + _ignoge_model_config_overrides = "_ignore_model_config_ui_overrides_" + _model_config_description = "The configuration of model number {}." + _model_config_description_note = ( + "Note that, when cloning this task and running it remotely," + " the configuration might be applied to another model instead of this one." + " To avoid this, initialize the task externally by calling `Task.init`" + " before the `ClearMLCallback` is instantiated." + ) + _train_run_counter = 0 + _model_connect_counter = 0 + _task_created_in_callback = False + _should_close_on_train_end = None + + def __init__(self): + if is_clearml_available(): + import clearml + + self._clearml = clearml + else: + raise RuntimeError("ClearMLCallback requires 'clearml' to be installed. Run `pip install clearml`.") + + self._initialized = False + self._clearml_task = None + + self._log_model = False + self._checkpoints_saved = [] + + def setup(self, args, state, model, processing_class, **kwargs): + if self._clearml is None: + return + if self._initialized: + return + ClearMLCallback._train_run_counter += 1 + ClearMLCallback._model_connect_counter += 1 + ClearMLCallback.log_suffix = ( + "" if ClearMLCallback._train_run_counter == 1 else "_" + str(ClearMLCallback._train_run_counter) + ) + if state.is_world_process_zero: + logger.info("Automatic ClearML logging enabled.") + if self._clearml_task is None: + if ClearMLCallback._should_close_on_train_end is None: + if not self._clearml.Task.running_locally() or self._clearml.Task.current_task(): + ClearMLCallback._should_close_on_train_end = False + else: + ClearMLCallback._should_close_on_train_end = True + + # This might happen when running inside of a pipeline, where the task is already initialized + # from outside of Hugging Face + if self._clearml.Task.running_locally() and self._clearml.Task.current_task(): + self._clearml_task = self._clearml.Task.current_task() + self._log_model = os.getenv( + "CLEARML_LOG_MODEL", + "FALSE" if not ClearMLCallback._task_created_in_callback else "TRUE", + ).upper() in ENV_VARS_TRUE_VALUES.union({"TRUE"}) + logger.info("External ClearML Task has been connected.") + else: + self._clearml_task = self._clearml.Task.init( + project_name=os.getenv("CLEARML_PROJECT", "HuggingFace Transformers"), + task_name=os.getenv("CLEARML_TASK", "Trainer"), + auto_connect_frameworks={"tensorboard": False, "pytorch": False}, + output_uri=True, + ) + self._log_model = os.getenv("CLEARML_LOG_MODEL", "TRUE").upper() in ENV_VARS_TRUE_VALUES.union( + {"TRUE"} + ) + ClearMLCallback._task_created_in_callback = True + logger.info("ClearML Task has been initialized.") + self._initialized = True + + suffixed_hparams_section = ClearMLCallback._hparams_section + ClearMLCallback.log_suffix + ignore_hparams_config_section = suffixed_hparams_section + "/" + ClearMLCallback._ignore_hparams_overrides + if self._clearml.Task.running_locally(): + self._copy_training_args_as_hparams(args, suffixed_hparams_section) + self._clearml_task.set_parameter( + name=ignore_hparams_config_section, + value=True, + value_type=bool, + description=( + "If True, ignore Transformers hyperparameters overrides done in the UI/backend " + + "when running remotely. Otherwise, the overrides will be applied when running remotely" + ), + ) + elif not self._clearml_task.get_parameter(ignore_hparams_config_section, default=True, cast=True): + self._clearml_task.connect(args, suffixed_hparams_section) + else: + self._copy_training_args_as_hparams( + args, ClearMLCallback._hparams_section + ClearMLCallback.log_suffix + ) + + if getattr(model, "config", None) is not None: + ignore_model_config_section = ( + suffixed_hparams_section + "/" + ClearMLCallback._ignoge_model_config_overrides + ) + configuration_object_description = ClearMLCallback._model_config_description.format( + ClearMLCallback._model_connect_counter + ) + if ClearMLCallback._model_connect_counter != ClearMLCallback._train_run_counter: + configuration_object_description += " " + ClearMLCallback._model_config_description_note + if self._clearml.Task.running_locally(): + self._clearml_task.set_parameter( + name=ignore_model_config_section, + value=True, + value_type=bool, + description=( + "If True, ignore Transformers model configuration overrides done in the UI/backend " + + "when running remotely. Otherwise, the overrides will be applied when running remotely" + ), + ) + self._clearml_task.set_configuration_object( + name=ClearMLCallback._model_config_section + ClearMLCallback.log_suffix, + config_dict=model.config.to_dict(), + description=configuration_object_description, + ) + elif not self._clearml_task.get_parameter(ignore_model_config_section, default=True, cast=True): + model.config = model.config.from_dict( + self._clearml_task.get_configuration_object_as_dict( + ClearMLCallback._model_config_section + ClearMLCallback.log_suffix + ) + ) + else: + self._clearml_task.set_configuration_object( + name=ClearMLCallback._model_config_section + ClearMLCallback.log_suffix, + config_dict=model.config.to_dict(), + description=configuration_object_description, + ) + + def on_train_begin(self, args, state, control, model=None, processing_class=None, **kwargs): + if self._clearml is None: + return + self._checkpoints_saved = [] + if state.is_hyper_param_search: + self._initialized = False + if not self._initialized: + self.setup(args, state, model, processing_class, **kwargs) + + def on_train_end(self, args, state, control, **kwargs): + if ClearMLCallback._should_close_on_train_end: + self._clearml_task.close() + ClearMLCallback._train_run_counter = 0 + + def on_log(self, args, state, control, model=None, processing_class=None, logs=None, **kwargs): + if self._clearml is None: + return + if not self._initialized: + self.setup(args, state, model, processing_class, **kwargs) + if state.is_world_process_zero: + eval_prefix = "eval_" + eval_prefix_len = len(eval_prefix) + test_prefix = "test_" + test_prefix_len = len(test_prefix) + single_value_scalars = [ + "train_runtime", + "train_samples_per_second", + "train_steps_per_second", + "train_loss", + "total_flos", + "epoch", + ] + for k, v in logs.items(): + if isinstance(v, (int, float)): + if k in single_value_scalars: + self._clearml_task.get_logger().report_single_value( + name=k + ClearMLCallback.log_suffix, value=v + ) + elif k.startswith(eval_prefix): + self._clearml_task.get_logger().report_scalar( + title="eval" + ClearMLCallback.log_suffix, + series=k[eval_prefix_len:], + value=v, + iteration=state.global_step, + ) + elif k.startswith(test_prefix): + self._clearml_task.get_logger().report_scalar( + title="test" + ClearMLCallback.log_suffix, + series=k[test_prefix_len:], + value=v, + iteration=state.global_step, + ) + else: + self._clearml_task.get_logger().report_scalar( + title="train" + ClearMLCallback.log_suffix, + series=k, + value=v, + iteration=state.global_step, + ) + else: + logger.warning( + "Trainer is attempting to log a value of " + f'"{v}" of type {type(v)} for key "{k}" as a scalar. ' + "This invocation of ClearML logger's report_scalar() " + "is incorrect so we dropped this attribute." + ) + + def on_save(self, args, state, control, **kwargs): + if self._log_model and self._clearml_task and state.is_world_process_zero: + ckpt_dir = f"checkpoint-{state.global_step}" + artifact_path = os.path.join(args.output_dir, ckpt_dir) + name = ckpt_dir + ClearMLCallback.log_suffix + logger.info(f"Logging checkpoint artifact `{name}`. This may take some time.") + output_model = self._clearml.OutputModel(task=self._clearml_task, name=name) + output_model.connect(task=self._clearml_task, name=name) + output_model.update_weights_package( + weights_path=artifact_path, + target_filename=ckpt_dir, + iteration=state.global_step, + auto_delete_file=False, + ) + self._checkpoints_saved.append(output_model) + while args.save_total_limit and args.save_total_limit < len(self._checkpoints_saved): + try: + self._clearml.model.Model.remove( + self._checkpoints_saved[0], + delete_weights_file=True, + force=True, + raise_on_errors=True, + ) + except Exception as e: + logger.warning( + "Could not remove checkpoint `{}` after going over the `save_total_limit`. Error is: {}".format( + self._checkpoints_saved[0].name, e + ) + ) + break + self._checkpoints_saved = self._checkpoints_saved[1:] + + def _copy_training_args_as_hparams(self, training_args, prefix): + as_dict = { + field.name: getattr(training_args, field.name) + for field in fields(training_args) + if field.init and not field.name.endswith("_token") + } + flat_dict = {str(k): v for k, v in self._clearml.utilities.proxy_object.flatten_dictionary(as_dict).items()} + self._clearml_task._arguments.copy_from_dict(flat_dict, prefix=prefix) + + +class FlyteCallback(TrainerCallback): + """A [`TrainerCallback`] that sends the logs to [Flyte](https://flyte.org/). + NOTE: This callback only works within a Flyte task. + + Args: + save_log_history (`bool`, *optional*, defaults to `True`): + When set to True, the training logs are saved as a Flyte Deck. + + sync_checkpoints (`bool`, *optional*, defaults to `True`): + When set to True, checkpoints are synced with Flyte and can be used to resume training in the case of an + interruption. + + Example: + + ```python + # Note: This example skips over some setup steps for brevity. + from flytekit import current_context, task + + + @task + def train_hf_transformer(): + cp = current_context().checkpoint + trainer = Trainer(..., callbacks=[FlyteCallback()]) + output = trainer.train(resume_from_checkpoint=cp.restore()) + ``` + """ + + def __init__(self, save_log_history: bool = True, sync_checkpoints: bool = True): + super().__init__() + if not is_flytekit_available(): + raise ImportError("FlyteCallback requires flytekit to be installed. Run `pip install flytekit`.") + + if not is_flyte_deck_standard_available() or not is_pandas_available(): + logger.warning( + "Syncing log history requires both flytekitplugins-deck-standard and pandas to be installed. " + "Run `pip install flytekitplugins-deck-standard pandas` to enable this feature." + ) + save_log_history = False + + from flytekit import current_context + + self.cp = current_context().checkpoint + self.save_log_history = save_log_history + self.sync_checkpoints = sync_checkpoints + + def on_save(self, args, state, control, **kwargs): + if self.sync_checkpoints and state.is_world_process_zero: + ckpt_dir = f"checkpoint-{state.global_step}" + artifact_path = os.path.join(args.output_dir, ckpt_dir) + + logger.info(f"Syncing checkpoint in {ckpt_dir} to Flyte. This may take time.") + self.cp.save(artifact_path) + + def on_train_end(self, args, state, control, **kwargs): + if self.save_log_history: + import pandas as pd + from flytekit import Deck + from flytekitplugins.deck.renderer import TableRenderer + + log_history_df = pd.DataFrame(state.log_history) + Deck("Log History", TableRenderer().to_html(log_history_df)) + + +class DVCLiveCallback(TrainerCallback): + """ + A [`TrainerCallback`] that sends the logs to [DVCLive](https://www.dvc.org/doc/dvclive). + + Use the environment variables below in `setup` to configure the integration. To customize this callback beyond + those environment variables, see [here](https://dvc.org/doc/dvclive/ml-frameworks/huggingface). + + Args: + live (`dvclive.Live`, *optional*, defaults to `None`): + Optional Live instance. If None, a new instance will be created using **kwargs. + log_model (Union[Literal["all"], bool], *optional*, defaults to `None`): + Whether to use `dvclive.Live.log_artifact()` to log checkpoints created by [`Trainer`]. If set to `True`, + the final checkpoint is logged at the end of training. If set to `"all"`, the entire + [`TrainingArguments`]'s `output_dir` is logged at each checkpoint. + """ + + def __init__( + self, + live: Optional[Any] = None, + log_model: Optional[Union[Literal["all"], bool]] = None, + **kwargs, + ): + if not is_dvclive_available(): + raise RuntimeError("DVCLiveCallback requires dvclive to be installed. Run `pip install dvclive`.") + from dvclive import Live + + self._initialized = False + self.live = None + if isinstance(live, Live): + self.live = live + elif live is not None: + raise RuntimeError(f"Found class {live.__class__} for live, expected dvclive.Live") + + self._log_model = log_model + if self._log_model is None: + log_model_env = os.getenv("HF_DVCLIVE_LOG_MODEL", "FALSE") + if log_model_env.upper() in ENV_VARS_TRUE_VALUES: + self._log_model = True + elif log_model_env.lower() == "all": + self._log_model = "all" + + def setup(self, args, state, model): + """ + Setup the optional DVCLive integration. To customize this callback beyond the environment variables below, see + [here](https://dvc.org/doc/dvclive/ml-frameworks/huggingface). + + Environment: + - **HF_DVCLIVE_LOG_MODEL** (`str`, *optional*): + Whether to use `dvclive.Live.log_artifact()` to log checkpoints created by [`Trainer`]. If set to `True` or + *1*, the final checkpoint is logged at the end of training. If set to `all`, the entire + [`TrainingArguments`]'s `output_dir` is logged at each checkpoint. + """ + from dvclive import Live + + self._initialized = True + if state.is_world_process_zero: + if not self.live: + self.live = Live() + self.live.log_params(args.to_dict()) + + def on_train_begin(self, args, state, control, model=None, **kwargs): + if not self._initialized: + self.setup(args, state, model) + + def on_log(self, args, state, control, model=None, logs=None, **kwargs): + if not self._initialized: + self.setup(args, state, model) + if state.is_world_process_zero: + from dvclive.plots import Metric + from dvclive.utils import standardize_metric_name + + for key, value in logs.items(): + if Metric.could_log(value): + self.live.log_metric(standardize_metric_name(key, "dvclive.huggingface"), value) + else: + logger.warning( + "Trainer is attempting to log a value of " + f'"{value}" of type {type(value)} for key "{key}" as a scalar. ' + "This invocation of DVCLive's Live.log_metric() " + "is incorrect so we dropped this attribute." + ) + self.live.next_step() + + def on_save(self, args, state, control, **kwargs): + if self._log_model == "all" and self._initialized and state.is_world_process_zero: + self.live.log_artifact(args.output_dir) + + def on_train_end(self, args, state, control, **kwargs): + if self._initialized and state.is_world_process_zero: + from transformers.trainer import Trainer + + if self._log_model is True: + fake_trainer = Trainer( + args=args, + model=kwargs.get("model"), + processing_class=kwargs.get("processing_class"), + eval_dataset=["fake"], + ) + name = "best" if args.load_best_model_at_end else "last" + output_dir = os.path.join(args.output_dir, name) + fake_trainer.save_model(output_dir) + self.live.log_artifact(output_dir, name=name, type="model", copy=True) + self.live.end() + + +class SwanLabCallback(TrainerCallback): + """ + A [`TrainerCallback`] that logs metrics, media, model checkpoints to [SwanLab](https://swanlab.cn/). + """ + + def __init__(self): + if not is_swanlab_available(): + raise RuntimeError("SwanLabCallback requires swanlab to be installed. Run `pip install swanlab`.") + import swanlab + + self._swanlab = swanlab + self._initialized = False + self._log_model = os.getenv("SWANLAB_LOG_MODEL", None) + + def setup(self, args, state, model, **kwargs): + """ + Setup the optional SwanLab (*swanlab*) integration. + + One can subclass and override this method to customize the setup if needed. Find more information + [here](https://docs.swanlab.cn/guide_cloud/integration/integration-huggingface-transformers.html). + + You can also override the following environment variables. Find more information about environment + variables [here](https://docs.swanlab.cn/en/api/environment-variable.html#environment-variables) + + Environment: + - **SWANLAB_API_KEY** (`str`, *optional*, defaults to `None`): + Cloud API Key. During login, this environment variable is checked first. If it doesn't exist, the system + checks if the user is already logged in. If not, the login process is initiated. + + - If a string is passed to the login interface, this environment variable is ignored. + - If the user is already logged in, this environment variable takes precedence over locally stored + login information. + + - **SWANLAB_PROJECT** (`str`, *optional*, defaults to `None`): + Set this to a custom string to store results in a different project. If not specified, the name of the current + running directory is used. + + - **SWANLAB_LOG_DIR** (`str`, *optional*, defaults to `swanlog`): + This environment variable specifies the storage path for log files when running in local mode. + By default, logs are saved in a folder named swanlog under the working directory. + + - **SWANLAB_MODE** (`Literal["local", "cloud", "disabled"]`, *optional*, defaults to `cloud`): + SwanLab's parsing mode, which involves callbacks registered by the operator. Currently, there are three modes: + local, cloud, and disabled. Note: Case-sensitive. Find more information + [here](https://docs.swanlab.cn/en/api/py-init.html#swanlab-init) + + - **SWANLAB_LOG_MODEL** (`str`, *optional*, defaults to `None`): + SwanLab does not currently support the save mode functionality.This feature will be available in a future + release + + - **SWANLAB_WEB_HOST** (`str`, *optional*, defaults to `None`): + Web address for the SwanLab cloud environment for private version (its free) + + - **SWANLAB_API_HOST** (`str`, *optional*, defaults to `None`): + API address for the SwanLab cloud environment for private version (its free) + + """ + self._initialized = True + + if state.is_world_process_zero: + logger.info('Automatic SwanLab logging enabled, to disable set os.environ["SWANLAB_MODE"] = "disabled"') + combined_dict = {**args.to_dict()} + + if hasattr(model, "config") and model.config is not None: + model_config = model.config if isinstance(model.config, dict) else model.config.to_dict() + combined_dict = {**model_config, **combined_dict} + if hasattr(model, "peft_config") and model.peft_config is not None: + peft_config = model.peft_config + combined_dict = {**{"peft_config": peft_config}, **combined_dict} + trial_name = state.trial_name + init_args = {} + if trial_name is not None: + init_args["experiment_name"] = f"{args.run_name}-{trial_name}" + elif args.run_name is not None: + init_args["experiment_name"] = args.run_name + init_args["project"] = os.getenv("SWANLAB_PROJECT", None) + + if self._swanlab.get_run() is None: + self._swanlab.init( + **init_args, + ) + # show transformers logo! + self._swanlab.config["FRAMEWORK"] = "🤗transformers" + # add config parameters (run may have been created manually) + self._swanlab.config.update(combined_dict) + + # add number of model parameters to swanlab config + try: + self._swanlab.config.update({"model_num_parameters": model.num_parameters()}) + # get peft model parameters + if type(model).__name__ == "PeftModel" or type(model).__name__ == "PeftMixedModel": + trainable_params, all_param = model.get_nb_trainable_parameters() + self._swanlab.config.update({"peft_model_trainable_params": trainable_params}) + self._swanlab.config.update({"peft_model_all_param": all_param}) + except AttributeError: + logger.info("Could not log the number of model parameters in SwanLab due to an AttributeError.") + + # log the initial model architecture to an artifact + if self._log_model is not None: + logger.warning( + "SwanLab does not currently support the save mode functionality. " + "This feature will be available in a future release." + ) + badge_markdown = ( + f'[Visualize in SwanLab]({self._swanlab.get_run().public.cloud.exp_url})' + ) + + modelcard.AUTOGENERATED_TRAINER_COMMENT += f"\n{badge_markdown}" + + def on_train_begin(self, args, state, control, model=None, **kwargs): + if not self._initialized: + self.setup(args, state, model, **kwargs) + + def on_train_end(self, args, state, control, model=None, processing_class=None, **kwargs): + if self._log_model is not None and self._initialized and state.is_world_process_zero: + logger.warning( + "SwanLab does not currently support the save mode functionality. " + "This feature will be available in a future release." + ) + + def on_log(self, args, state, control, model=None, logs=None, **kwargs): + single_value_scalars = [ + "train_runtime", + "train_samples_per_second", + "train_steps_per_second", + "train_loss", + "total_flos", + ] + + if not self._initialized: + self.setup(args, state, model) + if state.is_world_process_zero: + for k, v in logs.items(): + if k in single_value_scalars: + self._swanlab.log({f"single_value/{k}": v}, step=state.global_step) + non_scalar_logs = {k: v for k, v in logs.items() if k not in single_value_scalars} + non_scalar_logs = rewrite_logs(non_scalar_logs) + self._swanlab.log({**non_scalar_logs, "train/global_step": state.global_step}, step=state.global_step) + + def on_save(self, args, state, control, **kwargs): + if self._log_model is not None and self._initialized and state.is_world_process_zero: + logger.warning( + "SwanLab does not currently support the save mode functionality. " + "This feature will be available in a future release." + ) + + def on_predict(self, args, state, control, metrics, **kwargs): + if not self._initialized: + self.setup(args, state, **kwargs) + if state.is_world_process_zero: + metrics = rewrite_logs(metrics) + self._swanlab.log(metrics) + + +INTEGRATION_TO_CALLBACK = { + "azure_ml": AzureMLCallback, + "comet_ml": CometCallback, + "mlflow": MLflowCallback, + "neptune": NeptuneCallback, + "tensorboard": TensorBoardCallback, + "wandb": WandbCallback, + "codecarbon": CodeCarbonCallback, + "clearml": ClearMLCallback, + "dagshub": DagsHubCallback, + "flyte": FlyteCallback, + "dvclive": DVCLiveCallback, + "swanlab": SwanLabCallback, +} + + +def get_reporting_integration_callbacks(report_to): + if report_to is None: + return [] + + if isinstance(report_to, str): + if "none" == report_to: + return [] + elif "all" == report_to: + report_to = get_available_reporting_integrations() + else: + report_to = [report_to] + + for integration in report_to: + if integration not in INTEGRATION_TO_CALLBACK: + raise ValueError( + f"{integration} is not supported, only {', '.join(INTEGRATION_TO_CALLBACK.keys())} are supported." + ) + + return [INTEGRATION_TO_CALLBACK[integration] for integration in report_to] diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/logging.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..a2915e167a0bcf9d61aeaca269608b98320c3c54 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/logging.py @@ -0,0 +1,410 @@ +# Copyright 2020 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Logging utilities.""" + +import functools +import logging +import os +import sys +import threading +from logging import ( + CRITICAL, # NOQA + DEBUG, # NOQA + ERROR, # NOQA + FATAL, # NOQA + INFO, # NOQA + NOTSET, # NOQA + WARN, # NOQA + WARNING, # NOQA +) +from logging import captureWarnings as _captureWarnings +from typing import Optional + +import huggingface_hub.utils as hf_hub_utils +from tqdm import auto as tqdm_lib + + +_lock = threading.Lock() +_default_handler: Optional[logging.Handler] = None + +log_levels = { + "detail": logging.DEBUG, # will also print filename and line number + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + +_tqdm_active = not hf_hub_utils.are_progress_bars_disabled() + + +def _get_default_logging_level(): + """ + If TRANSFORMERS_VERBOSITY env var is set to one of the valid choices return that as the new default level. If it is + not - fall back to `_default_log_level` + """ + env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, " + f"has to be one of: {', '.join(log_levels.keys())}" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + global _default_handler + + with _lock: + if _default_handler: + # This library has already configured the library root logger. + return + _default_handler = logging.StreamHandler() # Set sys.stderr as stream. + # set defaults based on https://github.com/pyinstaller/pyinstaller/issues/7334#issuecomment-1357447176 + if sys.stderr is None: + sys.stderr = open(os.devnull, "w") + + _default_handler.flush = sys.stderr.flush + + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(_default_handler) + library_root_logger.setLevel(_get_default_logging_level()) + # if logging level is debug, we add pathname and lineno to formatter for easy debugging + if os.getenv("TRANSFORMERS_VERBOSITY", None) == "detail": + formatter = logging.Formatter("[%(levelname)s|%(pathname)s:%(lineno)s] %(asctime)s >> %(message)s") + _default_handler.setFormatter(formatter) + + is_ci = os.getenv("CI") is not None and os.getenv("CI").upper() in {"1", "ON", "YES", "TRUE"} + library_root_logger.propagate = True if is_ci else False + + +def _reset_library_root_logger() -> None: + global _default_handler + + with _lock: + if not _default_handler: + return + + library_root_logger = _get_library_root_logger() + library_root_logger.removeHandler(_default_handler) + library_root_logger.setLevel(logging.NOTSET) + _default_handler = None + + +def get_log_levels_dict(): + return log_levels + + +def captureWarnings(capture): + """ + Calls the `captureWarnings` method from the logging library to enable management of the warnings emitted by the + `warnings` library. + + Read more about this method here: + https://docs.python.org/3/library/logging.html#integration-with-the-warnings-module + + All warnings will be logged through the `py.warnings` logger. + + Careful: this method also adds a handler to this logger if it does not already have one, and updates the logging + level of that logger to the library's root logger. + """ + logger = get_logger("py.warnings") + + if not logger.handlers: + logger.addHandler(_default_handler) + + logger.setLevel(_get_library_root_logger().level) + + _captureWarnings(capture) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """ + Return a logger with the specified name. + + This function is not supposed to be directly accessed unless you are writing a custom transformers module. + """ + + if name is None: + name = _get_library_name() + + _configure_library_root_logger() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """ + Return the current level for the 🤗 Transformers's root logger as an int. + + Returns: + `int`: The logging level. + + + + 🤗 Transformers has following logging levels: + + - 50: `transformers.logging.CRITICAL` or `transformers.logging.FATAL` + - 40: `transformers.logging.ERROR` + - 30: `transformers.logging.WARNING` or `transformers.logging.WARN` + - 20: `transformers.logging.INFO` + - 10: `transformers.logging.DEBUG` + + """ + + _configure_library_root_logger() + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """ + Set the verbosity level for the 🤗 Transformers's root logger. + + Args: + verbosity (`int`): + Logging level, e.g., one of: + + - `transformers.logging.CRITICAL` or `transformers.logging.FATAL` + - `transformers.logging.ERROR` + - `transformers.logging.WARNING` or `transformers.logging.WARN` + - `transformers.logging.INFO` + - `transformers.logging.DEBUG` + """ + + _configure_library_root_logger() + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the verbosity to the `INFO` level.""" + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the verbosity to the `WARNING` level.""" + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the verbosity to the `DEBUG` level.""" + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the verbosity to the `ERROR` level.""" + return set_verbosity(ERROR) + + +def disable_default_handler() -> None: + """Disable the default handler of the HuggingFace Transformers's root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().removeHandler(_default_handler) + + +def enable_default_handler() -> None: + """Enable the default handler of the HuggingFace Transformers's root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().addHandler(_default_handler) + + +def add_handler(handler: logging.Handler) -> None: + """adds a handler to the HuggingFace Transformers's root logger.""" + + _configure_library_root_logger() + + assert handler is not None + _get_library_root_logger().addHandler(handler) + + +def remove_handler(handler: logging.Handler) -> None: + """removes given handler from the HuggingFace Transformers's root logger.""" + + _configure_library_root_logger() + + assert handler is not None and handler not in _get_library_root_logger().handlers + _get_library_root_logger().removeHandler(handler) + + +def disable_propagation() -> None: + """ + Disable propagation of the library log outputs. Note that log propagation is disabled by default. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """ + Enable propagation of the library log outputs. Please disable the HuggingFace Transformers's default handler to + prevent double logging if the root logger has been configured. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = True + + +def enable_explicit_format() -> None: + """ + Enable explicit formatting for every HuggingFace Transformers's logger. The explicit formatter is as follows: + ``` + [LEVELNAME|FILENAME|LINE NUMBER] TIME >> MESSAGE + ``` + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + formatter = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s") + handler.setFormatter(formatter) + + +def reset_format() -> None: + """ + Resets the formatting for HuggingFace Transformers's loggers. + + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + handler.setFormatter(None) + + +def warning_advice(self, *args, **kwargs): + """ + This method is identical to `logger.warning()`, but if env var TRANSFORMERS_NO_ADVISORY_WARNINGS=1 is set, this + warning will not be printed + """ + no_advisory_warnings = os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS", False) + if no_advisory_warnings: + return + self.warning(*args, **kwargs) + + +logging.Logger.warning_advice = warning_advice + + +@functools.lru_cache(None) +def warning_once(self, *args, **kwargs): + """ + This method is identical to `logger.warning()`, but will emit the warning with the same message only once + + Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache. + The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to + another type of cache that includes the caller frame information in the hashing function. + """ + self.warning(*args, **kwargs) + + +logging.Logger.warning_once = warning_once + + +@functools.lru_cache(None) +def info_once(self, *args, **kwargs): + """ + This method is identical to `logger.info()`, but will emit the info with the same message only once + + Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache. + The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to + another type of cache that includes the caller frame information in the hashing function. + """ + self.info(*args, **kwargs) + + +logging.Logger.info_once = info_once + + +class EmptyTqdm: + """Dummy tqdm which doesn't do anything.""" + + def __init__(self, *args, **kwargs): # pylint: disable=unused-argument + self._iterator = args[0] if args else None + + def __iter__(self): + return iter(self._iterator) + + def __getattr__(self, _): + """Return empty function.""" + + def empty_fn(*args, **kwargs): # pylint: disable=unused-argument + return + + return empty_fn + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + return + + +class _tqdm_cls: + def __call__(self, *args, **kwargs): + if _tqdm_active: + return tqdm_lib.tqdm(*args, **kwargs) + else: + return EmptyTqdm(*args, **kwargs) + + def set_lock(self, *args, **kwargs): + self._lock = None + if _tqdm_active: + return tqdm_lib.tqdm.set_lock(*args, **kwargs) + + def get_lock(self): + if _tqdm_active: + return tqdm_lib.tqdm.get_lock() + + +tqdm = _tqdm_cls() + + +def is_progress_bar_enabled() -> bool: + """Return a boolean indicating whether tqdm progress bars are enabled.""" + global _tqdm_active + return bool(_tqdm_active) + + +def enable_progress_bar(): + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = True + hf_hub_utils.enable_progress_bars() + + +def disable_progress_bar(): + """Disable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = False + hf_hub_utils.disable_progress_bars() diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/logits_process.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/logits_process.py new file mode 100644 index 0000000000000000000000000000000000000000..16c04478f08a64bbd1b65af5e720f1470940833b --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/logits_process.py @@ -0,0 +1,2942 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team and Google DeepMind. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from typing import Callable, Iterable, List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..pytorch_utils import isin_mps_friendly +from ..utils import add_start_docstrings +from ..utils.logging import get_logger + + +logger = get_logger(__name__) + + +LOGITS_PROCESSOR_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`): + Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam + search or log softmax for each vocabulary token when using beam search + + Return: + `torch.FloatTensor` of shape `(batch_size, config.vocab_size)`: The processed prediction scores. + +""" + + +class LogitsProcessor: + """Abstract base class for all logit processors that can be applied during generation.""" + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + raise NotImplementedError( + f"{self.__class__} is an abstract class. Only classes inheriting this class can be called." + ) + + +class LogitsProcessorList(list): + """ + This class can be used to create a list of [`LogitsProcessor`] to subsequently process a `scores` input tensor. + This class inherits from list and adds a specific *__call__* method to apply each [`LogitsProcessor`] to the + inputs. + """ + + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> torch.FloatTensor: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`): + Prediction scores of a language modeling head. These can be logits for each vocabulary when not using + beam search or log softmax for each vocabulary token when using beam search + kwargs (`Dict[str, Any]`, *optional*): + Additional kwargs that are specific to a logits processor. + + Return: + `torch.FloatTensor` of shape `(batch_size, config.vocab_size)`: + The processed prediction scores. + + """ + for processor in self: + function_args = inspect.signature(processor.__call__).parameters + if len(function_args) > 2: + if not all(arg in kwargs for arg in list(function_args.keys())[2:]): + raise ValueError( + f"Make sure that all the required parameters: {list(function_args.keys())} for " + f"{processor.__class__} are passed to the logits processor." + ) + scores = processor(input_ids, scores, **kwargs) + else: + scores = processor(input_ids, scores) + + return scores + + +class MinLengthLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] enforcing a min-length by setting EOS probability to 0. Note that, for decoder-only models + like most LLMs, the length includes the prompt. + + Args: + min_length (`int`): + The minimum length below which the score of `eos_token_id` is set to `-float("Inf")`. + eos_token_id (`Union[int, List[int], torch.Tensor]`): + The id(s) of the *end-of-sequence* token. + device (`str`, *optional*, defaults to `"cpu"`): + The device to allocate the tensors. + + Examples: + + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer + + >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m") + >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m") + + >>> inputs = tokenizer("A number:", return_tensors="pt") + >>> gen_out = model.generate(**inputs) + >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0]) + A number: one + + >>> # setting `min_length` to a value smaller than the uncontrolled output length has no impact + >>> gen_out = model.generate(**inputs, min_length=3) + >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0]) + A number: one + + >>> # setting a larger `min_length` will force the model to generate beyond its natural ending point, which is not + >>> # necessarily incorrect + >>> gen_out = model.generate(**inputs, min_length=10) + >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0]) + A number: one thousand, nine hundred and ninety-four + ``` + """ + + def __init__(self, min_length: int, eos_token_id: Union[int, List[int], torch.Tensor], device: str = "cpu"): + if not isinstance(min_length, int) or min_length < 0: + raise ValueError(f"`min_length` has to be a non-negative integer, but is {min_length}") + + if not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id, device=device) + + self.min_length = min_length + self.eos_token_id = eos_token_id + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + vocab_tensor = torch.arange(scores.shape[-1], device=scores.device) + eos_token_mask = isin_mps_friendly(vocab_tensor, self.eos_token_id) + scores_processed = scores.clone() + if input_ids.shape[-1] < self.min_length: + scores_processed = torch.where(eos_token_mask, -math.inf, scores) + return scores_processed + + +class MinNewTokensLengthLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] enforcing a min-length of new tokens by setting EOS (End-Of-Sequence) token probability to 0. + Contrarily to [`MinLengthLogitsProcessor`], this processor ignores the prompt. + + Args: + prompt_length_to_skip (`int`): + The input tokens length. Not a valid argument when used with `generate` as it will automatically assign the + input length. + min_new_tokens (`int`): + The minimum *new* tokens length below which the score of `eos_token_id` is set to `-float("Inf")`. + eos_token_id (`Union[int, List[int], torch.Tensor]`): + The id(s) of the *end-of-sequence* token. + device (`str`, *optional*, defaults to `"cpu"`): + The device to allocate the tensors. + + Examples: + + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer + + >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m") + >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m") + + >>> inputs = tokenizer(["A number:"], return_tensors="pt") + >>> gen_out = model.generate(**inputs) + >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0]) + A number: one + + >>> # setting `min_new_tokens` will force the model to generate beyond its natural ending point, which is not + >>> # necessarily incorrect + >>> gen_out = model.generate(**inputs, min_new_tokens=2) + >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0]) + A number: one thousand + ``` + """ + + def __init__( + self, + prompt_length_to_skip: int, + min_new_tokens: int, + eos_token_id: Union[int, List[int], torch.Tensor], + device: str = "cpu", + ): + for arg_name, arg_value in [ + ("prompt_length_to_skip", prompt_length_to_skip), + ("min_new_tokens", min_new_tokens), + ]: + if not isinstance(arg_value, int) or arg_value < 0: + raise ValueError(f"`{arg_name}` has to be a positive integer, but is {arg_value}") + + if not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id, device=device) + + self.prompt_length_to_skip = prompt_length_to_skip + self.min_new_tokens = min_new_tokens + self.eos_token_id = eos_token_id + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + new_tokens_length = input_ids.shape[-1] - self.prompt_length_to_skip + scores_processed = scores.clone() + vocab_tensor = torch.arange(scores.shape[-1], device=scores.device) + eos_token_mask = isin_mps_friendly(vocab_tensor, self.eos_token_id) + if new_tokens_length < self.min_new_tokens: + scores_processed = torch.where(eos_token_mask, -math.inf, scores) + + return scores_processed + + +class TemperatureLogitsWarper(LogitsProcessor): + r""" + [`LogitsProcessor`] for temperature (exponential scaling output probability distribution), which effectively means + that it can control the randomness of the predicted tokens. Often used together with [`TopPLogitsWarper`] and + [`TopKLogitsWarper`]. + + + + Make sure that `do_sample=True` is included in the `generate` arguments otherwise the temperature value won't have + any effect. + + + + Args: + temperature (`float`): + Strictly positive float value used to modulate the logits distribution. A value smaller than `1` decreases + randomness (and vice versa), with `0` being equivalent to shifting all probability mass to the most likely + token. + + Examples: + + ```python + >>> import torch + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> set_seed(0) # for reproducibility + + >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") + >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2") + >>> model.config.pad_token_id = model.config.eos_token_id + >>> inputs = tokenizer(["Hugging Face Company is"], return_tensors="pt") + + >>> # With temperature=1.0, the default, we consistently get random outputs due to random sampling. + >>> generate_kwargs = {"max_new_tokens": 10, "do_sample": True, "temperature": 1.0, "num_return_sequences": 2} + >>> outputs = model.generate(**inputs, **generate_kwargs) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)) + ['Hugging Face Company is one of these companies that is going to take a', + "Hugging Face Company is a brand created by Brian A. O'Neil"] + + >>> # However, with temperature close to 0, it approximates greedy decoding strategies (invariant) + >>> generate_kwargs["temperature"] = 0.0001 + >>> outputs = model.generate(**inputs, **generate_kwargs) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)) + ['Hugging Face Company is a company that has been around for over 20 years', + 'Hugging Face Company is a company that has been around for over 20 years'] + ``` + """ + + def __init__(self, temperature: float): + if not isinstance(temperature, float) or not (temperature > 0): + except_msg = ( + f"`temperature` (={temperature}) has to be a strictly positive float, otherwise your next token " + "scores will be invalid." + ) + if isinstance(temperature, float) and temperature == 0.0: + except_msg += " If you're looking for greedy decoding strategies, set `do_sample=False`." + raise ValueError(except_msg) + + self.temperature = temperature + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + scores_processed = scores / self.temperature + return scores_processed + + +class RepetitionPenaltyLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that prevents the repetition of previous tokens through a penalty. This penalty is applied at + most once per token. Note that, for decoder-only models like most LLMs, the considered tokens include the prompt. + + In the original [paper](https://arxiv.org/pdf/1909.05858.pdf), the authors suggest the use of a penalty of around + 1.2 to achieve a good balance between truthful generation and lack of repetition. To penalize and reduce + repetition, use `penalty` values above 1.0, where a higher value penalizes more strongly. To reward and encourage + repetition, use `penalty` values between 0.0 and 1.0, where a lower value rewards more strongly. + + Args: + penalty (`float`): + The parameter for repetition penalty. 1.0 means no penalty. Above 1.0 penalizes previously generated + tokens. Between 0.0 and 1.0 rewards previously generated tokens. + + Examples: + + ```py + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> # Initializing the model and tokenizer for it + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + >>> inputs = tokenizer(["I'm not going to"], return_tensors="pt") + + >>> # This shows a normal generate without any specific parameters + >>> summary_ids = model.generate(**inputs) + >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True)[0]) + I'm not going to be able to do that. I'm going to be able to do that + + >>> # This generates a penalty for repeated tokens + >>> penalized_ids = model.generate(**inputs, repetition_penalty=1.1) + >>> print(tokenizer.batch_decode(penalized_ids, skip_special_tokens=True)[0]) + I'm not going to be able to do that. I'll just have to go out and play + ``` + """ + + def __init__(self, penalty: float): + if not isinstance(penalty, float) or not (penalty > 0): + raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}") + + self.penalty = penalty + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + score = torch.gather(scores, 1, input_ids) + + # if score < 0 then repetition penalty has to be multiplied to reduce the token probabilities + score = torch.where(score < 0, score * self.penalty, score / self.penalty) + + scores_processed = scores.scatter(1, input_ids, score) + return scores_processed + + +class EncoderRepetitionPenaltyLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that works similarly to [`RepetitionPenaltyLogitsProcessor`], but with an *inverse* penalty + that is applied to the tokens present in the prompt. In other words, a penalty above 1.0 increases the odds of + selecting tokens that were present in the prompt. + + It was designed to avoid hallucination in input-grounded tasks, like summarization. Although originally intended + for encoder-decoder models, it can also be used with decoder-only models like LLMs. + + Args: + penalty (`float`): + The parameter for repetition penalty. 1.0 means no penalty. Above 1.0 rewards prompt tokens. Between 0.0 + and 1.0 penalizes prompt tokens. + encoder_input_ids (`torch.LongTensor`): + The encoder_input_ids that should be repeated within the decoder ids. + + Examples: + + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer + + >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m") + >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m") + + >>> inputs = tokenizer(["Alice and Bob. The third member's name was"], return_tensors="pt") + >>> gen_out = model.generate(**inputs) + >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0]) + Alice and Bob. The third member's name was not mentioned. + + >>> # With the `encoder_repetition_penalty` argument we can trigger this logits processor in `generate`, which can + >>> # promote the use of prompt tokens ("Bob" in this example) + >>> gen_out = model.generate(**inputs, encoder_repetition_penalty=1.2) + >>> print(tokenizer.batch_decode(gen_out, skip_special_tokens=True)[0]) + Alice and Bob. The third member's name was Bob. The third member's name was Bob. + ``` + """ + + def __init__(self, penalty: float, encoder_input_ids: torch.LongTensor): + if not isinstance(penalty, float) or not (penalty > 0): + raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}") + + self.penalty = 1 / penalty + self.encoder_input_ids = encoder_input_ids + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + score = torch.gather(scores, 1, self.encoder_input_ids) + + # if score < 0 then hallucination penalty has to be multiplied to increase the token probabilities + score = torch.where(score < 0, score * self.penalty, score / self.penalty) + + scores_processed = scores.scatter(1, self.encoder_input_ids, score) + return scores_processed + + +class TopPLogitsWarper(LogitsProcessor): + """ + [`LogitsProcessor`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off. + Often used together with [`TemperatureLogitsWarper`] and [`TopKLogitsWarper`]. + + Args: + top_p (`float`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + filter_value (`float`, *optional*, defaults to -inf): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> set_seed(1) + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + + >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt") + + >>> # With sampling, the output is unexpected -- sometimes too unexpected. + >>> outputs = model.generate(**inputs, do_sample=True) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3 | < 4 (left-hand pointer) ; + + + + >>> # With `top_p` sampling, the output gets restricted to high-probability tokens. + >>> # Pro tip: In practice, LLMs use `top_p` in the 0.9-0.95 range. + >>> outputs = model.generate(**inputs, do_sample=True, top_p=0.1) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9 + ``` + """ + + def __init__(self, top_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): + top_p = float(top_p) + if top_p < 0 or top_p > 1.0: + raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}") + if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1): + raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}") + + self.top_p = top_p + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + sorted_logits, sorted_indices = torch.sort(scores, descending=False) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Remove tokens with cumulative top_p above the threshold (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs <= (1 - self.top_p) + # Keep at least min_tokens_to_keep + sorted_indices_to_remove[..., -self.min_tokens_to_keep :] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) + scores_processed = scores.masked_fill(indices_to_remove, self.filter_value) + return scores_processed + + +class TopKLogitsWarper(LogitsProcessor): + r""" + [`LogitsProcessor`] that performs top-k, i.e. restricting to the k highest probability elements. Often used + together with [`TemperatureLogitsWarper`] and [`TopPLogitsWarper`]. + + Args: + top_k (`int`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + filter_value (`float`, *optional*, defaults to -inf): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> set_seed(1) + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + + >>> inputs = tokenizer("A sequence: A, B, C, D", return_tensors="pt") + + >>> # With sampling, the output is unexpected -- sometimes too unexpected. + >>> outputs = model.generate(**inputs, do_sample=True) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: A, B, C, D, E — S — O, P — R + + >>> # With `top_k` sampling, the output gets restricted the k most likely tokens. + >>> # Pro tip: In practice, LLMs use `top_k` in the 5-50 range. + >>> outputs = model.generate(**inputs, do_sample=True, top_k=2) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: A, B, C, D, E, F, G, H, I + ``` + """ + + def __init__(self, top_k: int, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): + if not isinstance(top_k, int) or top_k <= 0: + raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}") + + self.top_k = max(top_k, min_tokens_to_keep) + self.filter_value = filter_value + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + top_k = min(self.top_k, scores.size(-1)) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = scores < torch.topk(scores, top_k)[0][..., -1, None] + scores_processed = scores.masked_fill(indices_to_remove, self.filter_value) + return scores_processed + + +class MinPLogitsWarper(LogitsProcessor): + """ + [`LogitsProcessor`] that performs min-p, i.e. keeps all tokens that are above a minimum probability, scaled by the + probability of the most likely token. As a result, the filter becomes more agressive in the presence of + high-probability tokens, which is a sign of a confident output that we shouldn't deviate from. + + Often used together with [`TemperatureLogitsWarper`]. Used as an alternative to [`TopPLogitsWarper`] and + [`TopKLogitsWarper`]. + + Created by @menhguin and @kalomaze (github handles). Code adapted from [this external PR](https://github.com/oobabooga/text-generation-webui/pull/4449/files) + + Args: + min_p (`float`): + Minimum token probability, which will be scaled by the probability of the most likely token. It must be a + value between 0 and 1. Typical values are in the 0.01-0.2 range, comparably selective as setting `top_p` in + the 0.99-0.8 range (use the opposite of normal `top_p` values). + filter_value (`float`, *optional*, defaults to -inf): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> set_seed(1) + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + + >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt") + + >>> # With sampling, the output is unexpected -- sometimes too unexpected. + >>> outputs = model.generate(**inputs, do_sample=True) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3 | < 4 (left-hand pointer) ; + + + + >>> # With `min_p` sampling, the output gets restricted to high-probability tokens. + >>> # Pro tip: In practice, LLMs use `min_p` in the 0.01-0.2 range. + >>> outputs = model.generate(**inputs, do_sample=True, min_p=0.1) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9 + ``` + """ + + def __init__(self, min_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): + if not (0 <= min_p <= 1.0): + raise ValueError(f"`min_p` has to be a float in the [0, 1] interval, but is {min_p}") + if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1): + raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}") + + self.min_p = min_p + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # Convert logits to probabilities + probs = torch.softmax(scores, dim=-1) + # Get the probability of the top token for each sequence in the batch + top_probs, _ = probs.max(dim=-1, keepdim=True) + # Calculate the actual min_p threshold by scaling min_p with the top token's probability + scaled_min_p = self.min_p * top_probs + # Create a mask for tokens that have a probability less than the scaled min_p + tokens_to_remove = probs < scaled_min_p + + sorted_indices = torch.argsort(scores, descending=True, dim=-1) + sorted_indices_to_remove = torch.gather(tokens_to_remove, dim=-1, index=sorted_indices) + # Keep at least min_tokens_to_keep + sorted_indices_to_remove[..., : self.min_tokens_to_keep] = False + + indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) + scores_processed = scores.masked_fill(indices_to_remove, self.filter_value) + return scores_processed + + +class TypicalLogitsWarper(LogitsProcessor): + r""" + [`LogitsProcessor`] that performs typical decoding. Inspired on how humans use language, it prioritizes tokens + whose log probability is close to the entropy of the token probability distribution. This means that the most + likely tokens may be discarded in the process. + + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information. + + Args: + mass (`float`, *optional*, defaults to 0.9): + Value of typical_p between 0 and 1 inclusive, defaults to 0.9. + filter_value (`float`, *optional*, defaults to -inf): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m") + >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m") + + >>> inputs = tokenizer("1, 2, 3", return_tensors="pt") + + >>> # We can see that greedy decoding produces a sequence of numbers + >>> outputs = model.generate(**inputs) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + + >>> # For this particular seed, we can see that sampling produces nearly the same low-information (= low entropy) + >>> # sequence + >>> set_seed(18) + >>> outputs = model.generate(**inputs, do_sample=True) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + 1, 2, 3, 4, 5, 6, 7, 8, 9 and 10 + + >>> # With `typical_p` set, the most obvious sequence is no longer produced, which may be good for your problem + >>> set_seed(18) + >>> outputs = model.generate( + ... **inputs, do_sample=True, typical_p=0.1, return_dict_in_generate=True, output_scores=True + ... ) + >>> print(tokenizer.batch_decode(outputs.sequences, skip_special_tokens=True)[0]) + 1, 2, 3 and 5 + + >>> # We can see that the token corresponding to "4" (token 934) in the second position, the most likely token + >>> # as seen with greedy decoding, was entirely blocked out + >>> print(outputs.scores[1][0, 934]) + tensor(-inf) + ``` + """ + + def __init__(self, mass: float = 0.9, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): + mass = float(mass) + if not (mass > 0 and mass < 1): + raise ValueError(f"`typical_p` has to be a float > 0 and < 1, but is {mass}") + if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1): + raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}") + + self.filter_value = filter_value + self.mass = mass + self.min_tokens_to_keep = min_tokens_to_keep + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # calculate entropy + normalized = torch.nn.functional.log_softmax(scores, dim=-1) + p = torch.exp(normalized) + ent = -(normalized * p).nansum(-1, keepdim=True) + + # shift and sort + shifted_scores = torch.abs((-normalized) - ent) + sorted_scores, sorted_indices = torch.sort(shifted_scores, descending=False) + sorted_logits = scores.gather(-1, sorted_indices) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Remove tokens with cumulative mass above the threshold + last_ind = (cumulative_probs < self.mass).sum(dim=1) + last_ind.clamp_(max=sorted_scores.shape[-1] - 1) + sorted_indices_to_remove = sorted_scores > sorted_scores.gather(1, last_ind.view(-1, 1)) + sorted_indices_to_remove[..., : self.min_tokens_to_keep] = 0 + indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) + + scores_processed = scores.masked_fill(indices_to_remove, self.filter_value) + return scores_processed + + +class EpsilonLogitsWarper(LogitsProcessor): + r""" + [`LogitsProcessor`] that performs epsilon-sampling, i.e. restricting to tokens with `prob >= epsilon`. Takes the + largest min_tokens_to_keep tokens if no tokens satisfy this constraint. See [Truncation Sampling as Language Model + Desmoothing](https://arxiv.org/abs/2210.15191) for more information. + + Args: + epsilon (`float`): + If set to > 0, only the most tokens with probabilities `epsilon` or higher are kept for generation. + filter_value (`float`, *optional*, defaults to -inf): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + + Examples: + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> set_seed(1) + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + + >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt") + + >>> # With sampling, the output is unexpected -- sometimes too unexpected. + >>> outputs = model.generate(**inputs, do_sample=True) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3 | < 4 (left-hand pointer) ; + + + + >>> # With epsilon sampling, the output gets restricted to high-probability tokens. Note that this is similar to + >>> # Top P sampling, which restricts tokens based on their cumulative probability. + >>> # Pro tip: The paper recomends using `epsilon_cutoff` values between 3e-4 and 9e-4 + >>> outputs = model.generate(**inputs, do_sample=True, epsilon_cutoff=0.1) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9 + ``` + """ + + def __init__(self, epsilon: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): + epsilon = float(epsilon) + if epsilon <= 0 or epsilon >= 1: + raise ValueError(f"`epsilon_cutoff` has to be a float > 0 and < 1, but is {epsilon}") + + min_tokens_to_keep = int(min_tokens_to_keep) + if min_tokens_to_keep < 1: + raise ValueError( + f"`min_tokens_to_keep` has to be a strictly positive integer, but is {min_tokens_to_keep}" + ) + + self.epsilon = epsilon + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # Determine which indices to remove + probabilities = scores.softmax(dim=-1) + indices_to_remove = probabilities < self.epsilon + + # Keep the words with the 'min_tokens_to_keep'-highest probabilities + top_k = min(self.min_tokens_to_keep, scores.size(-1)) # Safety check + indices_to_remove = indices_to_remove & (scores < torch.topk(scores, top_k)[0][..., -1, None]) + + scores_processed = scores.masked_fill(indices_to_remove, self.filter_value) + return scores_processed + + +class EtaLogitsWarper(LogitsProcessor): + r""" + [`LogitsProcessor`] that performs eta-sampling, a technique to filter out tokens with probabilities below a dynamic + cutoff value, `eta`, which is calculated based on a combination of the hyperparameter `epsilon` and the entropy of + the token probabilities, i.e. `eta := min(epsilon, sqrt(epsilon * e^-entropy(probabilities)))`. Takes the largest + min_tokens_to_keep tokens if no tokens satisfy this constraint. It addresses the issue of poor quality in long + samples of text generated by neural language models leading to more coherent and fluent text. See [Truncation + Sampling as Language Model Desmoothing](https://arxiv.org/abs/2210.15191) for more information. Note: `do_sample` + must be set to `True` for this `LogitsProcessor` to work. + + + Args: + epsilon (`float`): + A float value in the range (0, 1). Hyperparameter used to calculate the dynamic cutoff value, `eta`. The + suggested values from the paper ranges from 3e-4 to 4e-3 depending on the size of the model. + filter_value (`float`, *optional*, defaults to -inf): + All values that are found to be below the dynamic cutoff value, `eta`, are set to this float value. This + parameter is useful when logits need to be modified for very low probability tokens that should be excluded + from generation entirely. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Specifies the minimum number of tokens that must be kept for generation, regardless of their probabilities. + For example, if `min_tokens_to_keep` is set to 1, at least one token will always be kept for generation, + even if all tokens have probabilities below the cutoff `eta`. + device (`str`, *optional*, defaults to `"cpu"`): + The device to allocate the tensors. + + Examples: + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> set_seed(1) + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + + >>> inputs = tokenizer("A sequence: 1, 2", return_tensors="pt") + + >>> # With sampling, the output is unexpected -- sometimes too unexpected. + >>> outputs = model.generate(**inputs, do_sample=True) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3 | < 4 (left-hand pointer) ; + + + + >>> # With eta sampling, the output gets restricted to high-probability tokens. You can see it as a dynamic form of + >>> # epsilon sampling that adapts its cutoff probability based on the entropy (high entropy = lower cutoff). + >>> # Pro tip: The paper recomends using `eta_cutoff` values between 3e-4 to 4e-3 + >>> outputs = model.generate(**inputs, do_sample=True, eta_cutoff=0.1) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + A sequence: 1, 2, 3, 4, 5, 6, 7, 8, 9 + ``` + """ + + def __init__( + self, epsilon: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1, device: str = "cpu" + ): + epsilon = float(epsilon) + if epsilon <= 0 or epsilon >= 1: + raise ValueError(f"`eta_cutoff` has to be a float > 0 and < 1, but is {epsilon}") + + min_tokens_to_keep = int(min_tokens_to_keep) + if min_tokens_to_keep < 1: + raise ValueError( + f"`min_tokens_to_keep` has to be a strictly positive integer, but is {min_tokens_to_keep}" + ) + + self.epsilon = torch.tensor(epsilon, device=device) + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + probabilities = scores.softmax(dim=-1) + entropy = torch.distributions.Categorical(logits=scores).entropy() + eta = torch.min(self.epsilon, torch.sqrt(self.epsilon) * torch.exp(-entropy))[..., None] + indices_to_remove = probabilities < eta + + # Keep the words with the 'min_tokens_to_keep'-highest probabilities + top_k = min(self.min_tokens_to_keep, scores.size(-1)) # Safety check + indices_to_remove = indices_to_remove & (scores < torch.topk(scores, top_k)[0][..., -1, None]) + + scores_processed = scores.masked_fill(indices_to_remove, self.filter_value) + return scores_processed + + +def _get_ngrams(ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int): + """ + Assume ngram_size=2 and prev_input_ids=tensor([[40, 2883, 2712, 4346]]). The output of generated ngrams look like + this {(40,): [2883], (2883,): [2712], (2712,): [4346]}. + + Args: + ngram_size (`int`): + The number sequential tokens taken as a group which may only occur once before being banned. + prev_input_ids (`torch.Tensor`): + Generated token ids for the current hypothesis. + num_hypos (`int`): + The number of hypotheses for which n-grams need to be generated. + + Returns: + generated_ngrams (`dict`): + Dictionary of generated ngrams. + """ + # Initialize an empty list of dictionaries, one for each hypothesis (index) in the range of num_hypos + generated_ngrams = [{} for _ in range(num_hypos)] + for idx in range(num_hypos): + gen_tokens = prev_input_ids[idx].tolist() + generated_ngram = generated_ngrams[idx] + # Loop through each n-gram of size ngram_size in the list of tokens (gen_tokens) + for ngram in zip(*[gen_tokens[i:] for i in range(ngram_size)]): + prev_ngram_tuple = tuple(ngram[:-1]) + generated_ngram[prev_ngram_tuple] = generated_ngram.get(prev_ngram_tuple, []) + [ngram[-1]] + return generated_ngrams + + +def _get_generated_ngrams(banned_ngrams, prev_input_ids, ngram_size, cur_len): + """ + Determines the banned tokens for the current hypothesis based on previously generated n-grams. + + Args: + banned_ngrams (`dict`): + A dictionary containing previously generated n-grams for each hypothesis. + prev_input_ids (`torch.Tensor`): + Generated token ids for the current hypothesis. + ngram_size (`int`): + The number sequential tokens taken as a group which may only occur once before being banned. + cur_len (`int`): + The current length of the token sequences for which the n-grams are being checked. + + Returns: + List of tokens that are banned. + """ + # Before decoding the next token, prevent decoding of ngrams that have already appeared + start_idx = cur_len + 1 - ngram_size + ngram_idx = tuple(prev_input_ids[start_idx:cur_len].tolist()) + return banned_ngrams.get(ngram_idx, []) + + +def _calc_banned_ngram_tokens( + ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int, cur_len: int +) -> List[Iterable[int]]: + """Copied from fairseq for no_repeat_ngram in beam_search""" + if cur_len + 1 < ngram_size: + # return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet + return [[] for _ in range(num_hypos)] + generated_ngrams = _get_ngrams(ngram_size, prev_input_ids, num_hypos) + banned_tokens = [ + _get_generated_ngrams(generated_ngrams[hypo_idx], prev_input_ids[hypo_idx], ngram_size, cur_len) + for hypo_idx in range(num_hypos) + ] + return banned_tokens + + +class NoRepeatNGramLogitsProcessor(LogitsProcessor): + r""" + N-grams are groups of "n" consecutive words, characters, or tokens taken from a sequence of text. Given the + sentence: "She runs fast", the bi-grams (n=2) would be ("she", "runs") and ("runs", "fast"). In text generation, + avoiding repetitions of word sequences provides a more diverse output. This [`LogitsProcessor`] enforces no + repetition of n-grams by setting the scores of banned tokens to negative infinity which eliminates those tokens + from consideration when further processing the scores. Note that, for decoder-only models like most LLMs, the + prompt is also considered to obtain the n-grams. + [Fairseq](https://github.com/pytorch/fairseq/blob/a07cb6f40480928c9e0548b737aadd36ee66ac76/fairseq/sequence_generator.py#L345). + + + + Use n-gram penalties with care. For instance, penalizing 2-grams (bigrams) in an article about the city of New York + might lead to undesirable outcomes where the city's name appears only once in the entire text. + [Reference](https://huggingface.co/blog/how-to-generate) + + + + Args: + ngram_size (`int`): + All ngrams of size `ngram_size` can only occur once. + + Examples: + + ```py + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + >>> inputs = tokenizer(["Today I"], return_tensors="pt") + + >>> output = model.generate(**inputs) + >>> print(tokenizer.decode(output[0], skip_special_tokens=True)) + Today I’m not sure if I’m going to be able to do it. + + >>> # Now let's add ngram size using `no_repeat_ngram_size`. This stops the repetitions ("I’m") in the output. + >>> output = model.generate(**inputs, no_repeat_ngram_size=2) + >>> print(tokenizer.decode(output[0], skip_special_tokens=True)) + Today I’m not sure if I can get a better understanding of the nature of this issue + ``` + """ + + def __init__(self, ngram_size: int): + if not isinstance(ngram_size, int) or ngram_size <= 0: + raise ValueError(f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}") + self.ngram_size = ngram_size + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + num_batch_hypotheses = scores.shape[0] + cur_len = input_ids.shape[-1] + scores_processed = scores.clone() + banned_batch_tokens = _calc_banned_ngram_tokens(self.ngram_size, input_ids, num_batch_hypotheses, cur_len) + for i, banned_tokens in enumerate(banned_batch_tokens): + scores_processed[i, banned_tokens] = -float("inf") + + return scores_processed + + +class EncoderNoRepeatNGramLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that works similarly to [`NoRepeatNGramLogitsProcessor`], but applied exclusively to prevent + the repetition of n-grams present in the prompt. + + It was designed to promote chattiness in a language model, by preventing the generation of n-grams present in + previous conversation rounds. + + Args: + encoder_ngram_size (`int`): + All ngrams of size `ngram_size` can only occur within the encoder input ids. + encoder_input_ids (`int`): + The encoder_input_ids that should not be repeated within the decoder ids. + + Examples: + + ```py + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m") + >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m") + + >>> inputs = tokenizer("Alice: I love cats. What do you love?\nBob:", return_tensors="pt") + + >>> # With greedy decoding, we see Bob repeating Alice's opinion. If Bob was a chatbot, it would be a poor one. + >>> outputs = model.generate(**inputs) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + Alice: I love cats. What do you love? + Bob: I love cats. What do you + + >>> # With this logits processor, we can prevent Bob from repeating Alice's opinion. + >>> outputs = model.generate(**inputs, encoder_no_repeat_ngram_size=2) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + Alice: I love cats. What do you love? + Bob: My cats are very cute. + ``` + """ + + def __init__(self, encoder_ngram_size: int, encoder_input_ids: torch.LongTensor): + if not isinstance(encoder_ngram_size, int) or encoder_ngram_size <= 0: + raise ValueError( + f"`encoder_ngram_size` has to be a strictly positive integer, but is {encoder_ngram_size}" + ) + self.ngram_size = encoder_ngram_size + if len(encoder_input_ids.shape) == 1: + encoder_input_ids = encoder_input_ids.unsqueeze(0) + self.batch_size = encoder_input_ids.shape[0] + self.generated_ngrams = _get_ngrams(encoder_ngram_size, encoder_input_ids, self.batch_size) + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # B x num_beams + num_hypos = scores.shape[0] + num_beams = num_hypos // self.batch_size + cur_len = input_ids.shape[-1] + scores_processed = scores.clone() + banned_batch_tokens = [ + _get_generated_ngrams( + self.generated_ngrams[hypo_idx // num_beams], input_ids[hypo_idx], self.ngram_size, cur_len + ) + for hypo_idx in range(num_hypos) + ] + + for i, banned_tokens in enumerate(banned_batch_tokens): + scores_processed[i, banned_tokens] = -float("inf") + + return scores_processed + + +class SequenceBiasLogitsProcessor(LogitsProcessor): + """ + [`LogitsProcessor`] that applies an additive bias on sequences. The bias is applied to the last token of a sequence + when the next generated token can complete it. Consequently, to take the most of biasing sequences with more than + one token, consider using beam methods (to gracefully work around partially completed sequences that have a + negative bias) and applying the bias to their prefixes (to ensure the bias is applied earlier). + + + + At a token-level, biasing a word is different from biasing a word with a space before it. If you want to bias + "foo" mid-sentence, you'll likely want to add a prefix space and bias " foo" instead. Check the tokenizer section + of our NLP course to find out why: https://huggingface.co/learn/nlp-course/chapter2/4?fw=pt + + + + Args: + sequence_bias (`List[List[Union[List[int], float]]]`): + List of lists that maps a sequence of tokens to its bias term (e.g. `[[[10, 45], -2.0], + [[64], -7.5]]`). Positive biases increase the odds of the + sequence being selected, while negative biases do the opposite. If a sequence has a length of 1, its bias + will always be applied. Otherwise, the bias will only be applied if the sequence in question is about to be + completed (in the token selection step after this processor is applied). + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct") + >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct") + >>> inputs = tokenizer(["The full name of Donald is Donald"], return_tensors="pt") + + >>> summary_ids = model.generate(inputs["input_ids"], max_new_tokens=4, do_sample=False) + >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True)[0]) + The full name of Donald is Donald John Trump Sr. + + >>> def get_tokens(word): + ... return tokenizer([word], add_special_tokens=False).input_ids[0] + + >>> # IMPORTANT: Remember our tip about adding spaces before words to bias them correctly. + >>> sequence_bias = [[get_tokens("Trump"), -10.0],] # will fail to apply bias + >>> biased_ids = model.generate( + ... inputs["input_ids"], max_new_tokens=4, do_sample=False, sequence_bias=sequence_bias + ... ) + >>> print(tokenizer.batch_decode(biased_ids, skip_special_tokens=True)[0]) + The full name of Donald is Donald John Trump Sr. + + >>> sequence_bias = [[get_tokens(" Trump"), -10.0],] # will work + >>> biased_ids = model.generate( + ... inputs["input_ids"], max_new_tokens=4, do_sample=False, sequence_bias=sequence_bias + ... ) + >>> print(tokenizer.batch_decode(biased_ids, skip_special_tokens=True)[0]) + The full name of Donald is Donald John Harper. He + + >>> # We can also add a positive bias to nudge the model towards specific tokens or continuations. This technique + >>> # is also more effective when paired up with beam search. + >>> sequence_bias = [[get_tokens(" Donald Duck"), 10.0],] + >>> biased_ids = model.generate( + ... inputs["input_ids"], max_new_tokens=4, num_beams=4, do_sample=False, sequence_bias=sequence_bias + ... ) + >>> print(tokenizer.batch_decode(biased_ids, skip_special_tokens=True)[0]) + The full name of Donald is Donald Duck. He is + ``` + """ + + def __init__(self, sequence_bias: List[List[Union[List[int], float]]]): + self.sequence_bias = sequence_bias + self._validate_arguments() + self._convert_list_arguments_into_dict() + + # Bias variables that will be populated on the first call (for retrocompatibility purposes, the vocabulary size + # is inferred in the first usage, which inhibits initializing here) + self.length_1_bias = None + self.prepared_bias_variables = False + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # 1 - Prepares the bias tensors. This is only needed the first time the logit processor is called. + if not self.prepared_bias_variables: + self._prepare_bias_variables(scores) + + # 2 - prepares an empty bias to add + bias = torch.zeros_like(scores) + + # 3 - include the bias from length = 1 + bias += self.length_1_bias + + # 4 - include the bias from length > 1, after determining which biased sequences may be completed. + for sequence_ids, sequence_bias in self.sequence_bias.items(): + if len(sequence_ids) == 1: # the sequence is of length 1, already applied + continue + if len(sequence_ids) > input_ids.shape[1]: # the sequence is longer than the context, ignore + continue + prefix_length = len(sequence_ids) - 1 + last_token = sequence_ids[-1] + matching_rows = torch.eq( + input_ids[:, -prefix_length:], + torch.tensor(sequence_ids[:-1], dtype=input_ids.dtype, device=input_ids.device), + ).prod(dim=1) + bias[:, last_token] += torch.where( + matching_rows.bool(), + torch.tensor(sequence_bias, device=input_ids.device), + torch.tensor(0.0, device=input_ids.device), + ) + + # 5 - apply the bias to the scores + scores_processed = scores + bias + return scores_processed + + def _prepare_bias_variables(self, scores: torch.FloatTensor): + vocabulary_size = scores.shape[-1] + + # Check biased tokens out of bounds + invalid_biases = [] + for sequence_ids in self.sequence_bias: + for token_id in sequence_ids: + if token_id >= vocabulary_size: + invalid_biases.append(token_id) + if len(invalid_biases) > 0: + raise ValueError( + f"The model vocabulary size is {vocabulary_size}, but the following tokens were being biased: " + f"{invalid_biases}" + ) + + # Precompute the bias tensors to be applied. Sequences of length 1 are kept separately, as they can be applied + # with simpler logic. + self.length_1_bias = torch.zeros((vocabulary_size,), dtype=torch.float, device=scores.device) + for sequence_ids, bias in self.sequence_bias.items(): + if len(sequence_ids) == 1: + self.length_1_bias[sequence_ids[-1]] = bias + + self.prepared_bias_variables = True + + def _validate_arguments(self): + sequence_bias = self.sequence_bias + if not isinstance(sequence_bias, dict) and not isinstance(sequence_bias, list) or len(sequence_bias) == 0: + raise ValueError( + f"`sequence_bias` has to be a non-empty dictionary, or non-empty list of lists but is {sequence_bias}." + ) + if isinstance(sequence_bias, dict) and any( + not isinstance(sequence_ids, tuple) for sequence_ids in sequence_bias.keys() + ): + raise ValueError(f"`sequence_bias` has to be a dict with tuples as keys, but is {sequence_bias}.") + if isinstance(sequence_bias, dict) and any( + any((not isinstance(token_id, (int, np.integer)) or token_id < 0) for token_id in sequence_ids) + or len(sequence_ids) == 0 + for sequence_ids in sequence_bias.keys() + ): + raise ValueError( + f"Each key in `sequence_bias` has to be a non-empty tuple of positive integers, but is " + f"{sequence_bias}." + ) + + def all_token_bias_pairs_are_valid(sequence): + return ( + isinstance(sequence[0], list) + and all(isinstance(token_id, (int, np.integer)) and token_id > 0 for token_id in sequence[0]) + and isinstance(sequence[1], float) + ) + + if isinstance(sequence_bias, list) and any( + (not all_token_bias_pairs_are_valid(sequence)) or len(sequence) == 0 for sequence in sequence_bias + ): + raise ValueError( + f"Each element in `sequence_bias` has to be a non-empty list of lists of positive integers and float, but is " + f"{sequence_bias}." + ) + if isinstance(sequence_bias, dict) and any(not isinstance(bias, float) for bias in sequence_bias.values()): + raise ValueError(f"`sequence_bias` has to be a dict with floats as values, but is {sequence_bias}.") + + def _convert_list_arguments_into_dict(self): + """BC: we used to accept `dict{tuple of tokens: float}` directly, now we expect a list""" + if isinstance(self.sequence_bias, list): + temp_sequence = self.sequence_bias + self.sequence_bias = {tuple(sublist[0]): sublist[1] for sublist in temp_sequence} + + +class NoBadWordsLogitsProcessor(SequenceBiasLogitsProcessor): + """ + [`LogitsProcessor`] that enforces that specified sequences will never be selected. + + + + In order to get the token ids of the words that should not appear in the generated text, make sure to set + `add_prefix_space=True` when initializing the tokenizer, and use `tokenizer(bad_words, + add_special_tokens=False).input_ids`. The `add_prefix_space` argument is only supported for some slow tokenizers, + as fast tokenizers' prefixing behaviours come from `pre tokenizers`. Read more + [here](https://huggingface.co/docs/tokenizers/api/pre-tokenizers). + + + + Args: + bad_words_ids (`List[List[int]]`): + List of list of token ids that are not allowed to be generated. + eos_token_id (`Union[int, List[int], torch.Tensor]`, *optional*): + The id(s) of the *end-of-sequence* token. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") + >>> inputs = tokenizer(["In a word, the cake is a"], return_tensors="pt") + + >>> output_ids = model.generate(inputs["input_ids"], max_new_tokens=5, pad_token_id=tokenizer.eos_token_id) + >>> print(tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]) + In a word, the cake is a bit of a mess. + + >>> # Now let's take the bad words out. Please note that the tokenizer is initialized differently + >>> tokenizer_with_prefix_space = AutoTokenizer.from_pretrained("openai-community/gpt2", add_prefix_space=True) + + + >>> def get_tokens_as_list(word_list): + ... "Converts a sequence of words into a list of tokens" + ... tokens_list = [] + ... for word in word_list: + ... tokenized_word = tokenizer_with_prefix_space([word], add_special_tokens=False).input_ids[0] + ... tokens_list.append(tokenized_word) + ... return tokens_list + + + >>> bad_words_ids = get_tokens_as_list(word_list=["mess"]) + >>> output_ids = model.generate( + ... inputs["input_ids"], max_new_tokens=5, bad_words_ids=bad_words_ids, pad_token_id=tokenizer.eos_token_id + ... ) + >>> print(tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]) + In a word, the cake is a bit of a surprise. + ``` + """ + + def __init__( + self, bad_words_ids: List[List[int]], eos_token_id: Optional[Union[int, List[int], torch.Tensor]] = None + ): + self.bad_word_ids = bad_words_ids + self._validate_arguments() + + # Filter EOS token from bad_words_ids + if eos_token_id is not None: + if not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id) + + bad_words_ids = list( + filter(lambda bad_token_seq: all(bad_token_seq != [i] for i in eos_token_id), bad_words_ids) + ) + + # Forbidding a sequence is equivalent to setting its bias to -inf + sequence_bias = {tuple(sequence): float("-inf") for sequence in bad_words_ids} + super().__init__(sequence_bias=sequence_bias) + + def _validate_arguments(self): + bad_words_ids = self.bad_word_ids + if not isinstance(bad_words_ids, list) or len(bad_words_ids) == 0: + raise ValueError(f"`bad_words_ids` has to be a non-empty list, but is {bad_words_ids}.") + if any(not isinstance(bad_word_ids, list) for bad_word_ids in bad_words_ids): + raise ValueError(f"`bad_words_ids` has to be a list of lists, but is {bad_words_ids}.") + if any( + any((not isinstance(token_id, (int, np.integer)) or token_id < 0) for token_id in bad_word_ids) + for bad_word_ids in bad_words_ids + ): + raise ValueError( + f"Each list in `bad_words_ids` has to be a list of positive integers, but is {bad_words_ids}." + ) + + +class PrefixConstrainedLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that enforces constrained generation and is useful for prefix-conditioned constrained + generation. See [Autoregressive Entity Retrieval](https://arxiv.org/abs/2010.00904) for more information. + + Args: + prefix_allowed_tokens_fn (`Callable[[int, torch.Tensor], List[int]]`): + This function constraints the beam search to allowed tokens only at each step. This function takes 2 + arguments `inputs_ids` and the batch ID `batch_id`. It has to return a list with the allowed tokens for the + next generation step conditioned on the previously generated tokens `inputs_ids` and the batch ID + `batch_id`. + + Examples: + + ```py + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-560m") + >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-560m") + + >>> inputs = tokenizer("Alice and Bob", return_tensors="pt") + + >>> # By default, it continues generating according to the model's logits + >>> outputs = model.generate(**inputs, max_new_tokens=5) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + Alice and Bob are friends + + >>> # We can contrain it with `prefix_allowed_tokens_fn` to force a certain behavior based on a prefix. + >>> # For instance, we can force an entire entity to be generated when its beginning is detected. + >>> entity = tokenizer(" Bob Marley", return_tensors="pt").input_ids[0] # 3 tokens + >>> def prefix_allowed_tokens_fn(batch_id, input_ids): + ... ''' + ... Attempts to generate 'Bob Marley' when 'Bob' is detected. + ... In this case, `batch_id` is not used, but you can set rules for each batch member. + ... ''' + ... if input_ids[-1] == entity[0]: + ... return [entity[1].item()] + ... elif input_ids[-2] == entity[0] and input_ids[-1] == entity[1]: + ... return [entity[2].item()] + ... return list(range(tokenizer.vocab_size)) # If no match, allow all tokens + + >>> outputs = model.generate(**inputs, max_new_tokens=5, prefix_allowed_tokens_fn=prefix_allowed_tokens_fn) + >>> print(tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]) + Alice and Bob Marley + ``` + """ + + def __init__(self, prefix_allowed_tokens_fn: Callable[[int, torch.Tensor], List[int]], num_beams: int): + self._prefix_allowed_tokens_fn = prefix_allowed_tokens_fn + self._num_beams = num_beams + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + mask = torch.full_like(scores, -math.inf) + batch_size = input_ids.shape[0] // self._num_beams + + for batch_id in range(batch_size): + for beam_id in range(self._num_beams): + sent = input_ids[batch_id * self._num_beams + beam_id] + prefix_allowed_tokens = self._prefix_allowed_tokens_fn(batch_id, sent) + if len(prefix_allowed_tokens) == 0: + raise ValueError( + f"`prefix_allowed_tokens_fn` returned an empty list for batch ID {batch_id}." + f"This means that the constraint is unsatisfiable. Please check your implementation" + f"of `prefix_allowed_tokens_fn` " + ) + mask[batch_id * self._num_beams + beam_id, prefix_allowed_tokens] = 0 + + scores_processed = scores + mask + return scores_processed + + +class HammingDiversityLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that enforces diverse beam search. + + Note that this logits processor is only effective for [`PreTrainedModel.group_beam_search`]. See [Diverse Beam + Search: Decoding Diverse Solutions from Neural Sequence Models](https://arxiv.org/pdf/1610.02424.pdf) for more + details. + + Traditional beam search often generates very similar sequences across different beams. + `HammingDiversityLogitsProcessor` addresses this by penalizing beams that generate tokens already chosen by other + beams in the same time step. + + Args: + diversity_penalty (`float`): + This value is subtracted from a beam's score if it generates a token same as any beam from other group at a + particular time. A higher `diversity_penalty` will enforce greater diversity among the beams. Adjusting + this value can help strike a balance between diversity and natural likelihood. + num_beams (`int`): + Number of beams for beam search. 1 means no beam search. + num_beam_groups (`int`): + Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams. + [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForSeq2SeqLM + >>> import torch + + >>> # Initialize the model and tokenizer + >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-base") + >>> model = AutoModelForSeq2SeqLM.from_pretrained("google-t5/t5-base") + + >>> # A long text about the solar system + >>> text = ( + ... "The Solar System is a gravitationally bound system comprising the Sun and the objects that orbit it, " + ... "either directly or indirectly. Of the objects that orbit the Sun directly, the largest are the eight " + ... "planets, with the remainder being smaller objects, such as the five dwarf planets and small Solar System " + ... "bodies. The Solar System formed 4.6 billion years ago from the gravitational collapse of a giant " + ... "interstellar molecular cloud." + ... ) + >>> inputs = tokenizer("summarize: " + text, return_tensors="pt") + + >>> # Generate diverse summary + >>> outputs_diverse = model.generate( + ... **inputs, + ... num_beam_groups=2, + ... diversity_penalty=10.0, + ... max_length=100, + ... num_beams=4, + ... num_return_sequences=2, + ... ) + >>> summaries_diverse = tokenizer.batch_decode(outputs_diverse, skip_special_tokens=True) + + >>> # Generate non-diverse summary + >>> outputs_non_diverse = model.generate( + ... **inputs, + ... max_length=100, + ... num_beams=4, + ... num_return_sequences=2, + ... ) + >>> summary_non_diverse = tokenizer.batch_decode(outputs_non_diverse, skip_special_tokens=True) + + >>> # With `diversity_penalty`, the resulting beams are much more diverse + >>> print(summary_non_diverse) + ['the solar system formed 4.6 billion years ago from the collapse of a giant interstellar molecular cloud. of the objects that orbit the Sun directly, the largest are the eight planets.', + 'the Solar System formed 4.6 billion years ago from the collapse of a giant interstellar molecular cloud. of the objects that orbit the Sun directly, the largest are the eight planets.'] + + >>> print(summaries_diverse) + ['the solar system formed 4.6 billion years ago from the collapse of a giant interstellar molecular cloud. of the objects that orbit the Sun directly, the largest are the eight planets.', + 'the solar system formed 4.6 billion years ago from the collapse of a giant interstellar molecular cloud. of the objects that orbit the Sun directly, the largest are the eight planets. the rest of the objects are smaller objects, such as the five dwarf planets and small solar system bodies.'] + ``` + """ + + def __init__(self, diversity_penalty: float, num_beams: int, num_beam_groups: int): + if not isinstance(diversity_penalty, float) or (not diversity_penalty > 0.0): + raise ValueError("`diversity_penalty` should be a float strictly larger than 0.") + self._diversity_penalty = diversity_penalty + if not isinstance(num_beams, int) or num_beams < 2: + raise ValueError("`num_beams` should be an integer strictly larger than 1.") + self._num_beams = num_beams + if not isinstance(num_beam_groups, int) or num_beam_groups < 2: + raise ValueError("`num_beam_groups` should be an integer strictly larger than 1.") + if num_beam_groups > num_beams: + raise ValueError("`beam_groups` has to be smaller or equal to `num_beams`.") + self._num_sub_beams = num_beams // num_beam_groups + + def __call__( + self, + input_ids: torch.LongTensor, + scores: torch.FloatTensor, + current_tokens: torch.LongTensor, + beam_group_idx: int, + ) -> torch.FloatTensor: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids) + scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`): + Prediction scores of a language modeling head. These can be logits for each vocabulary when not using + beam search or log softmax for each vocabulary token when using beam search + current_tokens (`torch.LongTensor` of shape `(batch_size)`): + Indices of input sequence tokens in the vocabulary, corresponding to the tokens selected by the other + beam groups in the current generation step. + beam_group_idx (`int`): + The index of the beam group currently being processed. + + Return: + `torch.FloatTensor` of shape `(batch_size, config.vocab_size)`: + The processed prediction scores. + """ + # hamming diversity: penalise using same token in current group which was used in previous groups at + # the same time step + batch_size = current_tokens.shape[0] // self._num_beams + group_start_idx = beam_group_idx * self._num_sub_beams + group_end_idx = min(group_start_idx + self._num_sub_beams, self._num_beams) + group_size = group_end_idx - group_start_idx + vocab_size = scores.shape[-1] + + if group_start_idx == 0: + return scores + + scores_processed = scores.clone() + for batch_idx in range(batch_size): + # predicted tokens of last time step of previous groups + previous_group_tokens = current_tokens[ + batch_idx * self._num_beams : batch_idx * self._num_beams + group_start_idx + ] + token_frequency = torch.bincount(previous_group_tokens, minlength=vocab_size).to(scores.device) + scores_processed[batch_idx * group_size : (batch_idx + 1) * group_size] -= ( + self._diversity_penalty * token_frequency + ) + + return scores_processed + + +class ForcedBOSTokenLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that enforces the specified token as the first generated token. Used with encoder-decoder + models. + + Args: + bos_token_id (`int`): + The id of the token to force as the first generated token. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForSeq2SeqLM + + >>> model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-small") + + >>> inputs = tokenizer("Translate from English to German: I love cats.", return_tensors="pt") + + >>> # By default, it continues generating according to the model's logits + >>> outputs = model.generate(**inputs, max_new_tokens=10) + >>> print(tokenizer.batch_decode(outputs)[0]) + Ich liebe Kitty. + + >>> # We can use `forced_bos_token_id` to force the start of generation with an encoder-decoder model + >>> # (including forcing it to end straight away with an EOS token) + >>> outputs = model.generate(**inputs, max_new_tokens=10, forced_bos_token_id=tokenizer.eos_token_id) + >>> print(tokenizer.batch_decode(outputs)[0]) + + ``` + """ + + def __init__(self, bos_token_id: int): + self.bos_token_id = bos_token_id + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + cur_len = input_ids.shape[-1] + scores_processed = scores + if cur_len == 1: + scores_processed = torch.full_like(scores, -math.inf) + scores_processed[:, self.bos_token_id] = 0 + return scores_processed + + +class ForcedEOSTokenLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that enforces the specified token as the last generated token when `max_length` is reached. + + Args: + max_length (`int`): + The maximum length of the sequence to be generated. + eos_token_id (`Union[int, List[int], torch.Tensor]`): + The id(s) of the *end-of-sequence* token. + device (`str`, *optional*, defaults to `"cpu"`): + The device to allocate the tensors. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + + >>> inputs = tokenizer("A sequence: 1, 2, 3", return_tensors="pt") + + >>> # By default, it continues generating according to the model's logits + >>> outputs = model.generate(**inputs, max_new_tokens=10) + >>> print(tokenizer.batch_decode(outputs)[0]) + A sequence: 1, 2, 3, 4, 5, 6, 7, 8 + + >>> # `forced_eos_token_id` ensures the generation ends with a EOS token + >>> outputs = model.generate(**inputs, max_new_tokens=10, forced_eos_token_id=tokenizer.eos_token_id) + >>> print(tokenizer.batch_decode(outputs)[0]) + A sequence: 1, 2, 3, 4, 5, 6, 7,<|endoftext|> + ``` + """ + + def __init__(self, max_length: int, eos_token_id: Union[int, List[int], torch.Tensor], device: str = "cpu"): + self.max_length = max_length + + if not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id, device=device) + self.eos_token_id = eos_token_id + + if torch.is_floating_point(eos_token_id) or (eos_token_id < 0).any(): + raise ValueError(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + cur_len = input_ids.shape[-1] + scores_processed = scores + if cur_len == self.max_length - 1: + scores_processed = torch.full_like(scores, -math.inf) + scores_processed[:, self.eos_token_id] = 0 + return scores_processed + + +class InfNanRemoveLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] that removes all `nan` and `inf` values to avoid the generation method to fail. Note that using + the logits processor should only be used if necessary since it can slow down the generation method. + + This logits processor has no `generate` example, as there shouldn't be a correct combination of flags that warrants + its use. + """ + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # set all nan values to 0.0 + scores_processed = torch.where(scores != scores, 0.0, scores) + + # set all +/-inf values to max/min possible value + scores_processed = torch.where(scores == float("inf"), torch.finfo(scores.dtype).max, scores_processed) + scores_processed = torch.where(scores == -float("inf"), torch.finfo(scores.dtype).min, scores_processed) + + return scores_processed + + +class ExponentialDecayLengthPenalty(LogitsProcessor): + r""" + [`LogitsProcessor`] that exponentially increases the score of the `eos_token_id` after `start_index` has been + reached. This allows generating shorter sequences without having a hard cutoff, allowing the `eos_token` to be + predicted in a meaningful position. + + Args: + exponential_decay_length_penalty (`tuple(int, float)`): + This tuple shall consist of: `(start_index, decay_factor)` where `start_index` indicates where penalty + starts and `decay_factor` represents the factor of exponential decay + eos_token_id (`Union[int, List[int], torch.Tensor]`): + The id(s) of the *end-of-sequence* token. + input_ids_seq_length (`int`): + The length of the input sequence. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed + + >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") + + >>> text = "Just wanted to let you know, I" + >>> inputs = tokenizer(text, return_tensors="pt") + + >>> # Let's consider that we want short sentences, so we limit `max_length=30`. However, we observe that the answer + >>> # tends to end abruptly. + >>> set_seed(1) + >>> outputs = model.generate(**inputs, do_sample=True, temperature=0.9, max_length=30, pad_token_id=50256) + >>> print(tokenizer.batch_decode(outputs)[0]) + Just wanted to let you know, I received a link to an ebook, the book How To Start A Social Network which was + published in 2010. Although + + >>> # To promote the appearance of the EOS token at the right time, we add the `exponential_decay_length_penalty = + >>> # (start_index, decay_factor)`. Instead of cutting at max_tokens, the output comes to an end before and usually + >>> # with more meaning. What happens is that starting from `start_index` the EOS token score will be increased + >>> # by `decay_factor` exponentially. However, if you set a high decay factor, you may also end up with abruptly + >>> # ending sequences. + >>> set_seed(1) + >>> outputs = model.generate( + ... **inputs, + ... do_sample=True, + ... temperature=0.9, + ... max_length=30, + ... pad_token_id=50256, + ... exponential_decay_length_penalty=(15, 1.6), + ... ) + >>> print(tokenizer.batch_decode(outputs)[0]) + Just wanted to let you know, I received a link to an ebook, the book How To Start A Social Network + which<|endoftext|> + + >>> # With a small decay factor, you will have a higher chance of getting a meaningful sequence. + >>> set_seed(1) + >>> outputs = model.generate( + ... **inputs, + ... do_sample=True, + ... temperature=0.9, + ... max_length=30, + ... pad_token_id=50256, + ... exponential_decay_length_penalty=(15, 1.01), + ... ) + >>> print(tokenizer.batch_decode(outputs)[0]) + Just wanted to let you know, I received a link to an ebook, the book How To Start A Social Network which was + published in 2010.<|endoftext|> + ``` + """ + + def __init__( + self, + exponential_decay_length_penalty: Tuple[int, float], + eos_token_id: Union[int, List[int], torch.Tensor], + input_ids_seq_length: int, + ): + self.regulation_start = exponential_decay_length_penalty[0] + input_ids_seq_length + self.regulation_factor = exponential_decay_length_penalty[1] + + if not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id) + self.eos_token_id = eos_token_id + + if torch.is_floating_point(eos_token_id) or (eos_token_id < 0).any(): + raise ValueError(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + cur_len = input_ids.shape[-1] + self.eos_token_id = self.eos_token_id.to(scores.device) + penalties = torch.zeros_like(scores) + scores_processed = scores + if cur_len > self.regulation_start: + penalty_idx = cur_len - self.regulation_start + # To support negative logits we compute the penalty of the absolute value and add to the original logit + penalty = torch.abs(scores[:, self.eos_token_id]) * (pow(self.regulation_factor, penalty_idx) - 1) + penalties[:, self.eos_token_id] = penalty + scores_processed = scores + penalties + return scores_processed + + +class LogitNormalization(LogitsProcessor): + r""" + [`LogitsProcessor`] for normalizing the scores using log-softmax. It's important to normalize + the scores during beam search, after applying the logits processors or warpers, since the search algorithm used in + this library doesn't do it (it only does it before, but they may need re-normalization) but it still supposes that + the scores are normalized when comparing the hypotheses. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + >>> import torch + + >>> model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert/distilgpt2") + + >>> inputs = tokenizer("A sequence: 1, 2, 3", return_tensors="pt") + + >>> # By default, the scores are not normalized -- the sum of their exponentials is NOT a normalized probability + >>> # distribution, summing to 1 + >>> outputs = model.generate(**inputs, return_dict_in_generate=True, output_scores=True) + >>> print(torch.allclose(torch.sum(torch.exp(outputs.scores[-1])), torch.Tensor((1.000,)), rtol=1e-4)) + False + + >>> # Normalizing them may have a positive impact on beam methods, or when using the scores on your application + >>> outputs = model.generate(**inputs, renormalize_logits=True, return_dict_in_generate=True, output_scores=True) + >>> print(torch.allclose(torch.sum(torch.exp(outputs.scores[-1])), torch.Tensor((1.000,)), rtol=1e-4)) + True + ``` + """ + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + scores_processed = scores.log_softmax(dim=-1) + return scores_processed + + +class SuppressTokensAtBeginLogitsProcessor(LogitsProcessor): + r""" + [`SuppressTokensAtBeginLogitsProcessor`] supresses a list of tokens as soon as the `generate` function starts + generating using `begin_index` tokens. This should ensure that the tokens defined by `begin_suppress_tokens` are + not generated at the beginning. Originally created for + [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper). + + Examples: + + ```python + >>> from transformers import AutoProcessor, WhisperForConditionalGeneration + >>> from datasets import load_dataset + + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt") + + >>> # Whisper has `begin_suppress_tokens` set by default (= `[220, 50256]`). 50256 is the EOS token, so this means + >>> # it can't generate and EOS token in the first iteration, but it can in the others. + >>> outputs = model.generate(**inputs, return_dict_in_generate=True, output_scores=True) + >>> print(outputs.scores[0][0, 50256]) + tensor(-inf) + >>> print(outputs.scores[-1][0, 50256]) # in other places we can see some probability mass for EOS + tensor(29.9010) + + >>> # If we disable `begin_suppress_tokens`, we can generate EOS in the first iteration. + >>> outputs = model.generate( + ... **inputs, return_dict_in_generate=True, output_scores=True, begin_suppress_tokens=None + ... ) + >>> print(outputs.scores[0][0, 50256]) + tensor(11.2027) + ``` + """ + + def __init__(self, begin_suppress_tokens, begin_index, device: str = "cpu"): + self.begin_suppress_tokens = torch.tensor(list(begin_suppress_tokens), device=device) + self.begin_index = begin_index + + def set_begin_index(self, begin_index): + self.begin_index = begin_index + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + vocab_tensor = torch.arange(scores.shape[-1], device=scores.device) + suppress_token_mask = isin_mps_friendly(vocab_tensor, self.begin_suppress_tokens) + scores_processed = scores + if input_ids.shape[-1] == self.begin_index: + scores_processed = torch.where(suppress_token_mask, -float("inf"), scores) + + return scores_processed + + +class SuppressTokensLogitsProcessor(LogitsProcessor): + r""" + This processor can be used to suppress a list of tokens. The processor will set their log probs to `-inf` so + that they are not generated. Originally created for + [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper). + + Examples: + + ```python + >>> from transformers import AutoProcessor, WhisperForConditionalGeneration + >>> from datasets import load_dataset + + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt") + + >>> # Whisper has a long list of suppressed tokens. For instance, in this case, the token 1 is suppressed by default. + >>> outputs = model.generate(**inputs, return_dict_in_generate=True, output_scores=True) + >>> print(outputs.scores[1][0, 1]) # 1 (and not 0) is the first freely generated token + tensor(-inf) + + >>> # If we disable `suppress_tokens`, we can generate it. + >>> outputs = model.generate(**inputs, return_dict_in_generate=True, output_scores=True, suppress_tokens=None) + >>> print(outputs.scores[1][0, 1]) + tensor(6.0678) + ``` + """ + + def __init__(self, suppress_tokens, device: str = "cpu"): + self.suppress_tokens = torch.tensor(list(suppress_tokens), device=device) + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + vocab_tensor = torch.arange(scores.shape[-1], device=scores.device) + suppress_token_mask = isin_mps_friendly(vocab_tensor, self.suppress_tokens) + scores = torch.where(suppress_token_mask, -float("inf"), scores) + return scores + + +class WhisperTimeStampLogitsProcessor(LogitsProcessor): + r""" + + [`LogitsProcessor`] that modifies the logits for the generation of timestamps in the transcription. When the input + tokens are at a specific threshold, the processor sets the scores to negative infinity. The processor makes sure + that timestamp tokens appear in pairs, by masking out the logits that would break this pairing pattern. This is + done to maintain the consistency and structure of generated timestamps. It also ensures that when the predicted + probability of sampling any of the timestamp token is greater than any individual non-timestamp token, those + non-timestamp logits are set to negative infinity. This is done to ensure the generation of timestamps over other + potential tokens. + + + See [the paper](https://arxiv.org/abs/2212.04356) for more information. + + Args: + generate_config (`GenerateConfig`): + The generate config used to generate the output. The following parameters are required: + eos_token_id (`int`, *optional*, defaults to 50257): + The id of the *end-of-sequence* token. + no_timestamps_token_id (`int`, *optional*, defaults to 50363): + The id of the `"<|notimestamps|>"` token. + max_initial_timestamp_index (`int`, *optional*, defaults to 1): + Used to set the maximum value of the initial timestamp. This is used to prevent the model from + predicting timestamps that are too far in the future. + begin_index (`Optional`, *optional*): Token index of the first token that is generated by the model. + _detect_timestamp_from_logprob (`bool`, *optional*): Whether timestamps can be predicted from logprobs over all timestamps. + + Examples: + ``` python + >>> import torch + >>> from transformers import AutoProcessor, WhisperForConditionalGeneration, GenerationConfig + >>> from datasets import load_dataset + + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> inputs = processor(ds[3]["audio"]["array"], return_tensors="pt") + >>> input_features = inputs.input_features + + >>> #Displaying timestamps + >>> generated_ids = model.generate(inputs=input_features, return_timestamps=True) + >>> transcription = processor.batch_decode(generated_ids, decode_with_timestamps=True)[0] + >>> print("Transcription:", transcription) + Transcription: <|startoftranscript|><|0.00|> He has grave doubts whether Sir Frederick Layton's work is really Greek after all, and can<|6.44|><|6.44|> discover in it but little of rocky Ithaca.<|9.44|><|endoftext|> + + + >>> #No timestamps & change EOS: + >>> #This allows the user to select a specific token to terminate the sequence on, in this case it's the word "can"(460) + >>> model.generation_config.eos_token_id = 460 + >>> generated_ids = model.generate(inputs=input_features,return_timestamps=False) + >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + >>> print("Transcription:", transcription) + Transcription: He has grave doubts whether Sir Frederick Layton's work is really Greek after all and can + ``` + """ + + def __init__( + self, + generate_config, + begin_index: Optional[int] = None, + _detect_timestamp_from_logprob: Optional[bool] = None, + ): # support for the kwargs + self.no_timestamps_token_id = generate_config.no_timestamps_token_id + self.timestamp_begin = generate_config.no_timestamps_token_id + 1 + self.eos_token_id = generate_config.eos_token_id or generate_config.bos_token_id + + # this variable is mostly just used for testing + self._detect_timestamp_from_logprob = ( + _detect_timestamp_from_logprob + if _detect_timestamp_from_logprob is not None + else getattr(generate_config, "_detect_timestamp_from_logprob", True) + ) + + num_forced_ids = ( + len(generate_config.forced_decoder_ids) if generate_config.forced_decoder_ids is not None else 0 + ) + self.begin_index = begin_index or (num_forced_ids + 1) + + self.max_initial_timestamp_index = getattr(generate_config, "max_initial_timestamp_index", None) + # TODO(Patrick): Make sure that official models have max_initial_timestamp_index set to 50 + # self.max_initial_timestamp_index = 50 + + def set_begin_index(self, begin_index): + self.begin_index = begin_index + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # suppress <|notimestamps|> which is handled by without_timestamps + scores_processed = scores.clone() + scores_processed[:, self.no_timestamps_token_id] = -float("inf") + + # timestamps have to appear in pairs, except directly before eos_token; mask logits accordingly + for k in range(input_ids.shape[0]): + sampled_tokens = input_ids[k, self.begin_index :] + seq = list(sampled_tokens.tolist()) + + last_was_timestamp = len(seq) >= 1 and seq[-1] >= self.timestamp_begin + penultimate_was_timestamp = len(seq) < 2 or seq[-2] >= self.timestamp_begin + + if last_was_timestamp: + if penultimate_was_timestamp: # has to be non-timestamp + scores_processed[k, self.timestamp_begin :] = -float("inf") + else: # cannot be normal text tokens + scores_processed[k, : self.eos_token_id] = -float("inf") + + timestamps = sampled_tokens[sampled_tokens.ge(self.timestamp_begin)] + if timestamps.numel() > 0: + # `timestamps` shouldn't decrease; forbid timestamp tokens smaller than the last + # The following lines of code are copied from: https://github.com/openai/whisper/pull/914/files#r1137085090 + if last_was_timestamp and not penultimate_was_timestamp: + timestamp_last = timestamps[-1] + else: + # Avoid to emit <|0.00|> again + timestamp_last = timestamps[-1] + 1 + + scores_processed[k, self.timestamp_begin : timestamp_last] = -float("inf") + + # apply the `max_initial_timestamp` option + if input_ids.shape[1] == self.begin_index: + scores_processed[:, : self.timestamp_begin] = -float("inf") + + if self.max_initial_timestamp_index is not None: + last_allowed = self.timestamp_begin + self.max_initial_timestamp_index + scores_processed[:, last_allowed + 1 :] = -float("inf") + + # if sum of probability over timestamps is above any other token, sample timestamp + logprobs = torch.nn.functional.log_softmax(scores_processed.float(), dim=-1) + for k in range(input_ids.shape[0]): + timestamp_logprob = logprobs[k, self.timestamp_begin :].logsumexp(dim=-1) + max_text_token_logprob = logprobs[k, : self.timestamp_begin].max() + if timestamp_logprob > max_text_token_logprob and self._detect_timestamp_from_logprob: + scores_processed[k, : self.timestamp_begin] = -float("inf") + + return scores_processed + + +class WhisperNoSpeechDetection(LogitsProcessor): + r"""This processor can be used to detect silence when using Whisper. It should take as input unprocessed logits to follow the original implementation""" + + def __init__(self, no_speech_token: int, begin_index: int, scores_is_logprobs: bool = False): + self.no_speech_token = no_speech_token + # offset between token, , in paper and first generated token + # is equal to the position of the first generated token index + self.start_of_trans_offset = begin_index + + # `self.begin_index` is a running value that is changed on the fly + self.begin_index = begin_index + self._no_speech_prob = [0.0] + self.is_scores_logprobs = scores_is_logprobs + + # overwritten dynamically + self.model = None + self.inputs = None + + def set_model(self, model): + self.model = model + + def set_inputs(self, inputs): + self.inputs = {**self.model.prepare_inputs_for_generation(**inputs), **inputs} + self.inputs["input_features"] = self.inputs.pop("inputs") + + @property + def no_speech_prob(self): + return self._no_speech_prob + + def set_begin_index(self, begin_index): + self.begin_index = begin_index + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + is_scores_logprobs = self.is_scores_logprobs + + if input_ids.shape[1] == self.begin_index: + if self.start_of_trans_offset > 1: + with torch.no_grad(): + logits = self.model(**self.inputs).logits + + no_speech_index = self.begin_index - self.start_of_trans_offset + no_speech_scores = logits[:, no_speech_index] + is_scores_logprobs = False + else: + no_speech_scores = scores + + if is_scores_logprobs: + probs = no_speech_scores.exp() + else: + probs = no_speech_scores.float().softmax(dim=-1) + + self._no_speech_prob = probs[:, self.no_speech_token] + + return scores + + +class ClassifierFreeGuidanceLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] for classifier free guidance (CFG). The scores are split over the batch dimension, + where the first half correspond to the conditional logits (predicted from the input prompt) and the second half + correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a + weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`. + + See [the paper](https://arxiv.org/abs/2306.05284) for more information. + + + + This logits processor is exclusively compatible with + [MusicGen](https://huggingface.co/docs/transformers/main/en/model_doc/musicgen) + + + + Args: + guidance_scale (float): + The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`. + Higher guidance scale encourages the model to generate samples that are more closely linked to the input + prompt, usually at the expense of poorer quality. + + Examples: + + ```python + >>> from transformers import AutoProcessor, MusicgenForConditionalGeneration + + >>> processor = AutoProcessor.from_pretrained("facebook/musicgen-small") + >>> model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small") + + >>> inputs = processor( + ... text=["80s pop track with bassy drums and synth", "90s rock song with loud guitars and heavy drums"], + ... padding=True, + ... return_tensors="pt", + ... ) + >>> audio_values = model.generate(**inputs, do_sample=True, guidance_scale=3, max_new_tokens=256) + ``` + """ + + def __init__(self, guidance_scale): + if guidance_scale > 1: + self.guidance_scale = guidance_scale + else: + raise ValueError( + "Require guidance scale >1 to use the classifier free guidance processor, got guidance scale " + f"{guidance_scale}." + ) + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + # simple check to make sure we have compatible batch sizes between our + # logits scores (cond + uncond) and input ids (cond only) + if scores.shape[0] != 2 * input_ids.shape[0]: + raise ValueError( + f"Logits should have twice the batch size of the input ids, the first half of batches corresponding to " + f"the conditional inputs, and the second half of batches corresponding to the unconditional inputs. Got " + f"batch size {scores.shape[0]} for the logits and {input_ids.shape[0]} for the input ids." + ) + unguided_bsz = scores.shape[0] // 2 + cond_logits, uncond_logits = scores.split(unguided_bsz, dim=0) + scores_processed = uncond_logits + (cond_logits - uncond_logits) * self.guidance_scale + return scores_processed + + +class AlternatingCodebooksLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] enforcing alternated generation between the two codebooks of Bark. + + + + This logits processor is exclusively compatible with + [Bark](https://huggingface.co/docs/transformers/en/model_doc/bark)'s fine submodel. See the model documentation + for examples. + + + + Args: + input_start_len (`int`): + The length of the initial input sequence. + semantic_vocab_size (`int`): + Vocabulary size of the semantic part, i.e number of tokens associated to the semantic vocabulary. + codebook_size (`int`): + Number of tokens associated to the codebook. + """ + + def __init__(self, input_start_len: int, semantic_vocab_size: int, codebook_size: int): + if not isinstance(input_start_len, int) or input_start_len < 0: + raise ValueError(f"`input_starting_length` has to be a non-negative integer, but is {input_start_len}") + + self.input_start_len = input_start_len + self.semantic_vocab_size = semantic_vocab_size + self.codebook_size = codebook_size + + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + curr_len = input_ids.shape[-1] + + # even -> first codebook, odd -> second codebook + is_first_codebook = ((curr_len - self.input_start_len) % 2) == 0 + + scores_processed = scores.clone() + if is_first_codebook: + scores_processed[:, : self.semantic_vocab_size] = -float("inf") + scores_processed[:, self.semantic_vocab_size + self.codebook_size :] = -float("inf") + else: + scores_processed[:, : self.semantic_vocab_size + self.codebook_size] = -float("inf") + + return scores_processed + + +class UnbatchedClassifierFreeGuidanceLogitsProcessor(LogitsProcessor): + r""" + Logits processor for Classifier-Free Guidance (CFG). The processors computes a weighted average across scores + from prompt conditional and prompt unconditional (or negative) logits, parameterized by the `guidance_scale`. + The unconditional scores are computed internally by prompting `model` with the `unconditional_ids` branch. + + See [the paper](https://arxiv.org/abs/2306.17806) for more information. + + Args: + guidance_scale (`float`): + The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale != 1`. + Higher guidance scale encourages the model to generate samples that are more closely linked to the input + prompt, usually at the expense of poorer quality. A value smaller than 1 has the opposite effect, while + making the negative prompt provided with negative_prompt_ids (if any) act as a positive prompt. + model (`PreTrainedModel`): + The model computing the unconditional scores. Supposedly the same as the one computing the conditional + scores. Both models must use the same tokenizer. + unconditional_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary for the unconditional branch. If unset, will default to + the last token of the prompt. + unconditional_attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Attention mask for unconditional_ids. + use_cache (`bool`, *optional*, defaults to `True`): + Whether to cache key/values during the negative prompt forward pass. + + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") + >>> inputs = tokenizer(["Today, a dragon flew over Paris, France,"], return_tensors="pt") + >>> out = model.generate(inputs["input_ids"], guidance_scale=1.5) + >>> tokenizer.batch_decode(out, skip_special_tokens=True)[0] + 'Today, a dragon flew over Paris, France, killing at least 50 people and injuring more than 100' + + >>> # with a negative prompt + >>> neg_inputs = tokenizer(["A very happy event happened,"], return_tensors="pt") + >>> out = model.generate(inputs["input_ids"], guidance_scale=2, negative_prompt_ids=neg_inputs["input_ids"]) + >>> tokenizer.batch_decode(out, skip_special_tokens=True)[0] + 'Today, a dragon flew over Paris, France, killing at least 130 people. French media reported that' + + >>> # with a positive prompt + >>> neg_inputs = tokenizer(["A very happy event happened,"], return_tensors="pt") + >>> out = model.generate(inputs["input_ids"], guidance_scale=0, negative_prompt_ids=neg_inputs["input_ids"]) + >>> tokenizer.batch_decode(out, skip_special_tokens=True)[0] + "Today, a dragon flew over Paris, France, and I'm very happy to be here. I" + ``` + """ + + def __init__( + self, + guidance_scale: float, + model, + unconditional_ids: Optional[torch.LongTensor] = None, + unconditional_attention_mask: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = True, + ): + self.guidance_scale = guidance_scale + self.model = model + self.unconditional_context = { + "input_ids": unconditional_ids, + "attention_mask": unconditional_attention_mask, + "use_cache": use_cache, + "past_key_values": None, + "first_pass": True, + } + + def get_unconditional_logits(self, input_ids): + if self.unconditional_context["first_pass"]: + if self.unconditional_context["input_ids"] is None: + self.unconditional_context["input_ids"] = input_ids[:, -1:] + if self.unconditional_context["attention_mask"] is None: + self.unconditional_context["attention_mask"] = torch.ones_like( + self.unconditional_context["input_ids"], dtype=torch.long + ) + input_ids = self.unconditional_context["input_ids"] + attention_mask = self.unconditional_context["attention_mask"] + self.unconditional_context["first_pass"] = False + else: + attention_mask = torch.cat( + [ + self.unconditional_context["attention_mask"], + torch.ones_like(input_ids[:, -1:], dtype=torch.long), + ], + dim=1, + ) + if not self.unconditional_context["use_cache"]: + input_ids = torch.cat([self.unconditional_context["input_ids"], input_ids[:, -1:]], dim=1) + else: + input_ids = input_ids[:, -1:] + self.unconditional_context["input_ids"] = input_ids + self.unconditional_context["attention_mask"] = attention_mask + + out = self.model( + input_ids, + attention_mask=attention_mask, + use_cache=self.unconditional_context["use_cache"], + past_key_values=self.unconditional_context["past_key_values"], + ) + self.unconditional_context["past_key_values"] = out.get("past_key_values", None) + + return out.logits + + def __call__(self, input_ids, scores): + scores = torch.nn.functional.log_softmax(scores, dim=-1) + if self.guidance_scale == 1: + return scores + + logits = self.get_unconditional_logits(input_ids) + + unconditional_logits = torch.nn.functional.log_softmax(logits[:, -1], dim=-1) + scores_processed = self.guidance_scale * (scores - unconditional_logits) + unconditional_logits + return scores_processed + + +class BarkEosPrioritizerLogitsProcessor(LogitsProcessor): + r"""This processor ensures that the EOS token is selected if its probability is greater than the `min_eos_p`. + + + + This logits processor is exclusively compatible with + [Bark](https://huggingface.co/docs/transformers/en/model_doc/bark). See the model documentation for examples. + + + + Args: + eos_token_id (`Union[int, List[int], torch.Tensor]`): + The id(s) of the *end-of-sequence* token. + min_eos_p (`float`, *optional*): + Minimum end of speech threshold. + """ + + def __init__(self, eos_token_id: Union[int, List[int], torch.Tensor], min_eos_p: float, device: str = "cpu"): + if not isinstance(eos_token_id, torch.Tensor): + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + eos_token_id = torch.tensor(eos_token_id, device=device) + self.eos_token_id = eos_token_id + + if torch.is_floating_point(eos_token_id) or (eos_token_id < 0).any(): + raise ValueError(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") + + if min_eos_p is not None and min_eos_p <= 0: + raise ValueError(f"`min_eos_p` has to be a positive float, but is {min_eos_p}") + self.min_eos_p = min_eos_p + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + scores_processed = scores + if self.min_eos_p: + probs = torch.nn.functional.softmax(scores.float(), dim=-1) + # create scores full of -inf except for the eos_token_id + early_stop_scores = torch.ones_like(scores) * -float("inf") + early_stop_scores[:, self.eos_token_id] = scores[:, self.eos_token_id] + + do_early_stop = probs[:, self.eos_token_id] > self.min_eos_p + do_early_stop = torch.any(do_early_stop, dim=1, keepdim=True) + scores_processed = torch.where(do_early_stop, early_stop_scores, scores) + + return scores_processed + + +class WatermarkLogitsProcessor(LogitsProcessor): + r""" + Logits processor for watermarking generated text. The processor modifies model output scores by adding a small bias to + randomized set of "green" tokens before generating the next token. "Green" tokens selection process depends on the + `seeding_scheme` used. The code was based on the [original repo](https://github.com/jwkirchenbauer/lm-watermarking/tree/main). + + The text generated by this `LogitsProcessor` can be detected using `WatermarkDetector`. See [`~WatermarkDetector.__call__`] for details, + + See [the paper](https://arxiv.org/abs/2306.04634) for more information. + + Args: + vocab_size (`int`): + The model tokenizer's vocab_size. Used to calculate "green" tokens ratio. + device (`str`): + The device where model is allocated. + greenlist_ratio (`float`, optional, *optional*, defaults to 0.25): + The ratio of "green" tokens used to the vocabulary size. Defaults to 0.25. + bias (`float`, optional, *optional*, defaults to 2.0): + The bias added to the selected "green" tokens' logits. Consider lowering the + `bias` if the text generation quality degrades. Recommended values are in the + range of [0.5, 2.0]. Defaults to 2.0. + hashing_key (`int`, optional, *optional*, defaults to 15485863): + Key used for hashing. If you deploy this watermark, we advise using another private key. + Defaults to 15485863 (the millionth prime). + seeding_scheme (`str`, optional, *optional*, defaults to `"lefthash"`): + The seeding scheme used for selecting "green" tokens. Accepts values: + - "lefthash" (default): "green" tokens selection depend on the last token (Algorithm 2 from paper) + - "selfhash": "green" tokens selection depends on the current token itself (Algorithm 3 from paper) + The downside of this scheme is that it considers all possible next tokens and can be slower than "lefthash". + The context length of previous tokens to use in seeding. Higher context length makes watermarking more robust. + context_width (`int`, *optional*, defaults to 1): + The number of previous tokens to use when setting the seed. + + Examples: + + ```python + >>> from transformers import AutoTokenizer, AutoModelForCausalLM, WatermarkingConfig + + >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") + >>> inputs = tokenizer(["Alice and Bob are"], return_tensors="pt") + + >>> # normal generation + >>> out = model.generate(inputs["input_ids"], max_length=20, do_sample=False) + >>> tokenizer.batch_decode(out, skip_special_tokens=True)[0] + 'Alice and Bob are both in the same room.\n\n"I\'m not sure if you\'re' + + >>> # watermarked generation + >>> watermarking_config = WatermarkingConfig(bias=2.5, context_width=2, seeding_scheme="selfhash") + >>> out = model.generate(inputs["input_ids"], watermarking_config=watermarking_config, max_length=20, do_sample=False) + >>> tokenizer.batch_decode(out, skip_special_tokens=True)[0] + 'Alice and Bob are both still alive and well and the story is pretty much a one-hour adventure' + + >>> # to detect watermarked text use the WatermarkDetector class + >>> from transformers import WatermarkDetector + >>> detector = WatermarkDetector(model_config=model.config, device="cpu", watermarking_config= watermarking_config) + >>> detection_preds = detector(out) + >>> detection_preds + array([ True]) + ``` + """ + + def __init__( + self, + vocab_size, + device, + greenlist_ratio: float = 0.25, + bias: float = 2.0, + hashing_key: int = 15485863, + seeding_scheme: str = "lefthash", + context_width: int = 1, + ): + if seeding_scheme not in ["selfhash", "lefthash"]: + raise ValueError(f"seeding_scheme has to be one of [`selfhash`, `lefthash`], but found {seeding_scheme}") + if greenlist_ratio >= 1.0 or greenlist_ratio <= 0.0: + raise ValueError( + f"greenlist_ratio has be in range between 0.0 and 1.0, exclusively. but found {greenlist_ratio}" + ) + + self.vocab_size = vocab_size + self.greenlist_size = int(self.vocab_size * greenlist_ratio) + self.bias = bias + self.seeding_scheme = seeding_scheme + self.rng = torch.Generator(device=device) + self.hash_key = hashing_key + self.context_width = context_width + + self.rng.manual_seed(hashing_key) + self.table_size = 1_000_003 + self.fixed_table = torch.randperm(self.table_size, generator=self.rng, device=device) + + def set_seed(self, input_seq: torch.LongTensor): + input_seq = input_seq[-self.context_width :] + if self.seeding_scheme == "selfhash": + a = self.fixed_table[input_seq % self.table_size] + 1 + b = self.fixed_table[input_seq[-1] % self.table_size] + 1 + seed = (self.hash_key * a * b).min().item() + else: + seed = self.hash_key * input_seq[-1].item() + self.rng.manual_seed(seed % (2**64 - 1)) + + def _get_greenlist_ids(self, input_seq: torch.LongTensor) -> torch.LongTensor: + self.set_seed(input_seq) + vocab_permutation = torch.randperm(self.vocab_size, device=input_seq.device, generator=self.rng) + greenlist_ids = vocab_permutation[: self.greenlist_size] + return greenlist_ids + + def _score_rejection_sampling(self, input_seq: torch.LongTensor, scores: torch.FloatTensor) -> torch.LongTensor: + """ + Generate greenlist based on current candidate next token. Reject and move on if necessary. + Runs for a fixed number of steps only for efficiency, since the methods is not batched. + """ + final_greenlist = [] + _, greedy_predictions = scores.sort(dim=-1, descending=True) + + # 40 is an arbitrary number chosen to save compute and not run for long (taken from orig repo) + for i in range(40): + greenlist_ids = self._get_greenlist_ids(torch.cat([input_seq, greedy_predictions[i, None]], dim=-1)) + if greedy_predictions[i] in greenlist_ids: + final_greenlist.append(greedy_predictions[i]) + return torch.tensor(final_greenlist, device=input_seq.device) + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + if input_ids.shape[-1] < self.context_width: + logger.warning( + f"`input_ids` should have at least `{self.context_width}` tokens but has {input_ids.shape[-1]}. " + "The seeding will be skipped for this generation step!" + ) + return scores + + scores_processed = scores.clone() + for b_idx, input_seq in enumerate(input_ids): + if self.seeding_scheme == "selfhash": + greenlist_ids = self._score_rejection_sampling(input_seq, scores[b_idx]) + else: + greenlist_ids = self._get_greenlist_ids(input_seq) + scores_processed[b_idx, greenlist_ids] = scores_processed[b_idx, greenlist_ids] + self.bias + + return scores_processed + + +class SynthIDTextWatermarkState: + """SynthID watermarking state.""" + + def __init__( + self, + batch_size: int, + ngram_len: int, + context_history_size: int, + device: torch.device, + ): + """Initializes the state. + + Args: + batch_size (`int`): Batch size. + ngram_len (`int`): Ngram length. + context_history_size (`int`): Size of the tensor to keep track of seen contexts. + device (`int`): Device to use. + """ + self.context = torch.zeros( + (batch_size, ngram_len - 1), + dtype=torch.int64, + device=device, + ) + self.context_history = torch.zeros( + (batch_size, context_history_size), + dtype=torch.int64, + device=device, + ) + self.num_calls = 0 + + +class SynthIDTextWatermarkLogitsProcessor(LogitsProcessor): + r""" + Logits processor that implements watermarking techniques for text generation models. + This class facilitates the application of SynthID text watermarking, a method for embedding imperceptible signals + into generated text to aid in detecting synthetic content. It operates by subtly manipulating the probabilities of + token selection during text generation in a manner that can be reliably recovered later for verification. + + Key Features: + * **State Management:** Maintains internal state to track token sequences and generate watermarking keys + dynamically. + + * **Key Generation:** Computes hashes based on token sequences and watermarking parameters to create unique keys + for each position. + + * **G-Value Sampling:** Employs a pre-computed sampling table to sample watermarking values (g-values) based on + the generated keys. + + * **Score Adjustment:** Applies calculated g-values to modify token probabilities during generation, embedding the + watermark. + + * **Context Repetition Handling:** Incorporates logic to avoid watermarking tokens in repeated contexts, + preserving naturalness. + + * **EOS Token Masking:** Supports masking end-of-sentence tokens to prevent their inclusion in watermarking + calculations. + + * **Utility Functions:** Provides functions to compute g-values directly, check for context repetition, create + EOS token masks, and estimate expected mean g-values. + + Refer to paper url: https://www.nature.com/articles/s41586-024-08025-4 for more details around this. + + Args: + ngram_len (`int`): + Ngram length. + keys (`List[int]`): + A sequence of watermarking keys, one for each depth. + sampling_table_size (`int`): + Size of the sampling table. + sampling_table_seed (`int`): + Random seed to generate the sampling table. + context_history_size (`int`): + Size of the tensor to keep track of seen contexts. + device (`torch.device`): + Device to use. + skip_first_ngram_calls (`bool`, *optional*, defaults to `False`): + Whether to skip first ngram calls. + debug_mode (`bool`, optional, *optional*, defaults to `False`): + Logits are modified to uniform one got before watermarking modification is applied. This is to test the + implementation. + + Examples: + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer, SynthIDTextWatermarkingConfig + + >>> tokenizer = AutoTokenizer.from_pretrained('google/gemma-2-2b', padding_side="left") + >>> model = AutoModelForCausalLM.from_pretrained('google/gemma-2-2b') + + >>> # SynthID Text configuration + >>> watermarking_config = SynthIDTextWatermarkingConfig( + ... keys=[654, 400, 836, 123, 340, 443, 597, 160, 57], + ... ngram_len=5, + ... ) + + >>> # Generation with watermarking + >>> tokenized_prompts = tokenizer(["Once upon a time, "], return_tensors="pt", padding=True) + >>> output_sequences = model.generate( + ... **tokenized_prompts, watermarking_config=watermarking_config, do_sample=True, max_new_tokens=10 + ... ) + >>> watermarked_text = tokenizer.batch_decode(output_sequences, skip_special_tokens=True) + ``` + """ + + def __init__( + self, + ngram_len: int, + keys: List[int], + sampling_table_size: int, + sampling_table_seed: int, + context_history_size: int, + device: torch.device, + skip_first_ngram_calls: bool = False, + debug_mode: bool = False, + ): + self.ngram_len = ngram_len + self.keys = torch.tensor(keys, device=device) + + generator = torch.Generator(device=device).manual_seed(sampling_table_seed) + # A random sampling table is pre-computed and modulo table size is applied to map from a hash of ngram keys to + # g values, this is similar to the hashtable implementation used in + # https://github.com/facebookresearch/three_bricks. We note that the hashing employed in this repository is + # different from that used to watermark the Gemini App, and hence the detectors trained based on the + # hashing in this repository will not transfer to text generated by the Gemini App. + self.sampling_table = torch.randint( + low=0, + high=2, + size=(sampling_table_size,), + generator=generator, + device=device, + ) + self.context_history_size = context_history_size + self.device = device + self.state = None + self.skip_first_ngram_calls = skip_first_ngram_calls + self.debug_mode = debug_mode + + def _init_state(self, batch_size: int): + """Initializes the state.""" + self.state = SynthIDTextWatermarkState( + batch_size=batch_size, + ngram_len=self.ngram_len, + context_history_size=self.context_history_size, + device=self.device, + ) + + def update_scores(self, scores: torch.FloatTensor, g_values: torch.FloatTensor) -> torch.FloatTensor: + """Updates scores using the g values. + + We assume that the scores are in the log space. + Args: + scores (`torch.FloatTensor`): Scores (batch_size, vocab_size). + g_values (`torch.FloatTensor`): G valus (batch_size, vocab_size, depth). + + Returns: + Updated scores (batch_size, vocab_size). + """ + _, _, depth = g_values.shape + + probs = torch.softmax(scores, dim=1) + + for i in range(depth): + g_values_at_depth = g_values[:, :, i] + g_mass_at_depth = (g_values_at_depth * probs).sum(axis=1, keepdims=True) + probs = probs * (1 + g_values_at_depth - g_mass_at_depth) + + log_probs = torch.log(probs) + log_probs = torch.where(torch.isfinite(log_probs), log_probs, torch.finfo(log_probs.dtype).min) + return log_probs + + @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING) + def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + self._check_input_ids_shape(input_ids) + batch_size, vocab_size = scores.shape + + if self.debug_mode: + scores = torch.ones_like(scores) + + # Currently indices is just a arange to compute watermarking on the desnse logits. + all_indices = torch.stack([torch.arange(vocab_size, device=self.device) for _ in range(batch_size)]) + + if self.state is None: + # Initialize watermarking state if it does not exist. + self._init_state(batch_size) + else: + # Append last input id (which is the input id added in last call) to the + # previous context so we have the context to be used for current + # watermarking. + self.state.context = torch.concat( + (self.state.context, input_ids[:, -1:]), + dim=1, + ) + self.state.context = self.state.context[:, 1:] + + if self.state is None: + raise ValueError("self.state can't be None! Call `self._init_state` to initialize the state.") + + self.state.num_calls += 1 + + # Don't watermark the first ngram_len - 1 tokens if set. + if self.skip_first_ngram_calls and self.state.num_calls < self.ngram_len: + return scores + + # 2. Generate random keys for each ngram key combination. + ngram_keys, hash_result_with_just_context = self._compute_keys(self.state.context, all_indices) + # ngram_keys shape [batch_size, top_k, depth] + + # 3. Sample g values. + g_values = self.sample_g_values(ngram_keys) + # g_values shape [batch_size, top_k, depth] + + # 4. Modify scores. + updated_scores = self.update_scores(scores, g_values) + # updated scores shape [batch_size, top_k] + + # 5. Check if the current watermarking context was previously used, if yes skip watermarking. + hash_result_with_just_context = hash_result_with_just_context[:, None] + is_repeated_context = (self.state.context_history == hash_result_with_just_context).any( + dim=1, + keepdim=True, + ) + self.state.context_history = torch.concat( + (hash_result_with_just_context, self.state.context_history), + dim=1, + )[:, :-1] + + updated_watermarked_scores = torch.where( + is_repeated_context, + input=scores, + other=updated_scores, + ) + return updated_watermarked_scores + + def accumulate_hash( + self, + current_hash: torch.LongTensor, + data: torch.LongTensor, + multiplier: int = 6364136223846793005, + increment: int = 1, + ) -> torch.LongTensor: + """ + Accumulate hash of data on current hash. + + Method uses adapted linear congruential generator with newlib/musl parameters. + + This function has following property - + f(x, data[T]) = f(f(x, data[:T - 1]), data[T]) + + This function expects current_hash.shape and data.shape[:-1] to + match/broadcastable. + + Args: + current_hash (`torch.LongTensor`): + (shape,) + data (`torch.LongTensor`): + (shape, tensor_len) + multiplier (`int`, optional, *optional*, defaults to 6364136223846793005): + multiplier of linear congruential generator + increment (`int`, optional, *optional*, defaults to 1): + increment of linear congruential generator + + Returns: + updated hash (shape,) + """ + for i in range(data.shape[-1]): + current_hash = torch.add(current_hash, data[..., i]) + current_hash = torch.mul(current_hash, multiplier) + current_hash = torch.add(current_hash, increment) + return current_hash + + def compute_ngram_keys(self, ngrams: torch.LongTensor) -> torch.LongTensor: + """Computes random keys for each ngram and depth. + + Args: + ngrams (`torch.LongTensor`): + Ngrams (batch_size, num_ngrams, ngram_len). + + Returns: + ngram keys (batch_size, num_ngrams, depth). + """ + if len(ngrams.shape) != 3: + raise ValueError(f"Ngrams should be of shape (batch_size, num_ngrams, ngram_len), but is {ngrams.shape}") + if ngrams.shape[2] != self.ngram_len: + raise ValueError( + "Ngrams should be of shape (batch_size, num_ngrams, ngram_len)," + f" where ngram_len is {self.ngram_len}, but is {ngrams.shape}" + ) + batch_size, _, _ = ngrams.shape + + hash_result = torch.ones(batch_size, device=self.device, dtype=torch.long) + # hash_result shape [batch_size,] + # ngrams shape [batch_size, num_ngrams, ngram_len] + hash_result = torch.vmap(self.accumulate_hash, in_dims=(None, 1), out_dims=1)(hash_result, ngrams) + # hash_result shape [batch_size, num_ngrams] + + keys = self.keys[None, None, :, None] + # hash_result shape [batch_size, num_ngrams] + # keys shape [1, 1, depth, 1] + hash_result = torch.vmap(self.accumulate_hash, in_dims=(None, 2), out_dims=2)(hash_result, keys) + # hash_result shape [batch_size, num_ngrams, depth] + + return hash_result + + def _compute_keys( + self, n_minus_1_grams: torch.LongTensor, indices: torch.LongTensor + ) -> Tuple[torch.LongTensor, torch.LongTensor]: + """Computes random keys for each ngram and depth. + + Args: + n_minus_1_grams (`torch.LongTensor`): + Ngrams (batch_size, ngram_len - 1). + indices (`torch.LongTensor`): + indices of the continuations (batch_size, num_indices) + + Returns: + Ngram keys (batch_size, num_indices, depth). + """ + batch_size, _ = n_minus_1_grams.shape + + hash_result = torch.ones(batch_size, device=self.device, dtype=torch.long) + # First hash n_minus_1 gram, for each batch entry we have a single + # n_minus_1 gram context. + # hash_result shape [batch_size] + # n_minus_1_gram shape [batch_size, ngram_len - 1] + hash_result_with_just_context = self.accumulate_hash(hash_result, n_minus_1_grams) + # hash_result shape [batch_size,] + # Indices is of shape [batch_size, num_indices], so we make it + # [batch_size, num_indices, 1] so we can vmap over num_indices dim. + hash_result = torch.vmap(self.accumulate_hash, in_dims=(None, 1), out_dims=1)( + hash_result_with_just_context, indices[:, :, None] + ) + # hash_result shape [batch_size, num_indices] + # Basically we have a hash for each batch entry and each indices + # Now we add watermarking keys to this hash. + # keys are of shape [depth,] + # We add batch, num_indices and data dimension to this making it + # [1, 1, depth, 1]. + # So we can vmap over the depth dimension for compute_hash + keys = self.keys[None, None, :, None] + hash_result = torch.vmap(self.accumulate_hash, in_dims=(None, 2), out_dims=2)(hash_result, keys) + # hash_result shape should be [batch_size, num_indices, depth] + return hash_result, hash_result_with_just_context + + def sample_g_values(self, ngram_keys: torch.LongTensor) -> torch.LongTensor: + """ + Samples g values from Bernoulli distribution. + + It is not possible to pass random keys in a vectorized way in torch. Instead + we pre-compute a random sampling table, and use apply modulo table size to + map from ngram keys (int64) to g values. + + Args: + ngram_keys (`torch.LongTensor`): + Random keys (batch_size, num_ngrams, depth). + + Returns: + G values (batch_size, num_ngrams, depth). + """ + (sampling_table_size,) = self.sampling_table.shape + sampling_table = self.sampling_table.reshape((1, 1, sampling_table_size)) + ngram_keys = ngram_keys % sampling_table_size + return torch.take_along_dim(sampling_table, indices=ngram_keys, dim=2) + + def _check_input_ids_shape(self, input_ids: torch.LongTensor): + """Checks the shape of input ids.""" + if len(input_ids.shape) != 2: + raise ValueError(f"Input ids should be of shape (batch_size, input_len), but is {input_ids.shape}") + + def compute_g_values(self, input_ids: torch.LongTensor) -> torch.LongTensor: + """ + Computes g values for each ngram from the given sequence of tokens. + + Args: + input_ids (`torch.LongTensor`): + Input token ids (batch_size, input_len). + + Returns: + G values (batch_size, input_len - (ngram_len - 1), depth). + """ + self._check_input_ids_shape(input_ids) + ngrams = input_ids.unfold(dimension=1, size=self.ngram_len, step=1) + ngram_keys = self.compute_ngram_keys(ngrams) + return self.sample_g_values(ngram_keys) + + def compute_context_repetition_mask(self, input_ids: torch.LongTensor) -> torch.LongTensor: + """ + Computes repetition mask. + + 0 and 1 stand for repeated and not repeated context n-1 grams respectively. + + Args: + input_ids (`torch.LongTensor`): + Input token ids (batch_size, input_len). + + Returns: + Repetitions mask (batch_size, input_len - (ngram_len - 1)). + """ + self._check_input_ids_shape(input_ids) + batch_size, _ = input_ids.shape + state = SynthIDTextWatermarkState( + batch_size=batch_size, + ngram_len=self.ngram_len, + context_history_size=self.context_history_size, + device=self.device, + ) + contexts = input_ids[:, :-1].unfold( + dimension=1, + size=self.ngram_len - 1, + step=1, + ) + _, num_contexts, _ = contexts.shape + + are_repeated_contexts = [] + for i in range(num_contexts): + context = contexts[:, i, :] + hash_result = torch.ones(batch_size, device=self.device, dtype=torch.long) + context_hash = self.accumulate_hash(hash_result, context)[:, None] + is_repeated_context = (state.context_history == context_hash).any( + dim=1, + keepdim=True, + ) + are_repeated_contexts.append(is_repeated_context) + state.context_history = torch.concat( + (context_hash, state.context_history), + dim=1, + )[:, :-1] + are_repeated_contexts = torch.concat(are_repeated_contexts, dim=1) + + return torch.logical_not(are_repeated_contexts) + + def compute_eos_token_mask(self, input_ids: torch.LongTensor, eos_token_id: int) -> torch.LongTensor: + """ + Computes repetitions mask. + + 1 stands for ngrams that don't contain EOS tokens and vice versa. + + Args: + input_ids (`torch.LongTensor`): + Input token ids (batch_size, input_len). + eos_token_id (`int`): + EOS token ID. + + Returns: + EOS token mask (batch_size, input_len). + """ + self._check_input_ids_shape(input_ids) + noneos_masks = [] + all_eos_equated = input_ids == eos_token_id + for eos_equated in all_eos_equated: + nonzero_idx = torch.nonzero(eos_equated) + noneos_mask = torch.ones_like(eos_equated) + if nonzero_idx.shape[0] != 0: + noneos_mask[nonzero_idx[0][0] :] = 0 + noneos_masks.append(noneos_mask) + return torch.stack(noneos_masks, dim=0) + + def expected_mean_g_value(self, vocab_size: int, coinflip_prob: float = 0.5) -> float: + """ + Compute expected mean g-value after watermarking, assuming uniform LM dist. + + This is the theoretical expected value for single-layer watermarking. + + Args: + vocab_size (`int`): + The size of the vocabulary. + coinflip_prob arg_name (`float`, *optional*, defaults to 0.5): + Probability of 1 in boolean prf. + + Returns: + The expected mean g-value for watermarked text. + """ + return coinflip_prob + coinflip_prob * (1 - coinflip_prob) * (1 - (1 / vocab_size)) diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_deformable_detr.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_deformable_detr.py new file mode 100644 index 0000000000000000000000000000000000000000..62080bcb3fd94fc0fc9051640744a81e03657d4e --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_deformable_detr.py @@ -0,0 +1,178 @@ +import torch +import torch.nn as nn + +from ..image_transforms import center_to_corners_format +from ..utils import is_scipy_available +from .loss_for_object_detection import ( + HungarianMatcher, + ImageLoss, + _set_aux_loss, + generalized_box_iou, + sigmoid_focal_loss, +) + + +if is_scipy_available(): + from scipy.optimize import linear_sum_assignment + + +class DeformableDetrHungarianMatcher(HungarianMatcher): + @torch.no_grad() + def forward(self, outputs, targets): + """ + Differences: + - out_prob = outputs["logits"].flatten(0, 1).sigmoid() instead of softmax + - class_cost uses alpha and gamma + """ + batch_size, num_queries = outputs["logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + out_prob = outputs["logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] + out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] + + # Also concat the target labels and boxes + target_ids = torch.cat([v["class_labels"] for v in targets]) + target_bbox = torch.cat([v["boxes"] for v in targets]) + + # Compute the classification cost. + alpha = 0.25 + gamma = 2.0 + neg_cost_class = (1 - alpha) * (out_prob**gamma) * (-(1 - out_prob + 1e-8).log()) + pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) + class_cost = pos_cost_class[:, target_ids] - neg_cost_class[:, target_ids] + + # Compute the L1 cost between boxes + bbox_cost = torch.cdist(out_bbox, target_bbox, p=1) + + # Compute the giou cost between boxes + giou_cost = -generalized_box_iou(center_to_corners_format(out_bbox), center_to_corners_format(target_bbox)) + + # Final cost matrix + cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost + cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() + + sizes = [len(v["boxes"]) for v in targets] + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))] + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +class DeformableDetrImageLoss(ImageLoss): + def __init__(self, matcher, num_classes, focal_alpha, losses): + nn.Module.__init__(self) + self.matcher = matcher + self.num_classes = num_classes + self.focal_alpha = focal_alpha + self.losses = losses + + # removed logging parameter, which was part of the original implementation + def loss_labels(self, outputs, targets, indices, num_boxes): + """ + Classification loss (Binary focal loss) targets dicts must contain the key "class_labels" containing a tensor + of dim [nb_target_boxes] + """ + if "logits" not in outputs: + raise KeyError("No logits were found in the outputs") + source_logits = outputs["logits"] + + idx = self._get_source_permutation_idx(indices) + target_classes_o = torch.cat([t["class_labels"][J] for t, (_, J) in zip(targets, indices)]) + target_classes = torch.full( + source_logits.shape[:2], self.num_classes, dtype=torch.int64, device=source_logits.device + ) + target_classes[idx] = target_classes_o + + target_classes_onehot = torch.zeros( + [source_logits.shape[0], source_logits.shape[1], source_logits.shape[2] + 1], + dtype=source_logits.dtype, + layout=source_logits.layout, + device=source_logits.device, + ) + target_classes_onehot.scatter_(2, target_classes.unsqueeze(-1), 1) + + target_classes_onehot = target_classes_onehot[:, :, :-1] + loss_ce = ( + sigmoid_focal_loss(source_logits, target_classes_onehot, num_boxes, alpha=self.focal_alpha, gamma=2) + * source_logits.shape[1] + ) + losses = {"loss_ce": loss_ce} + + return losses + + +def DeformableDetrForSegmentationLoss( + logits, labels, device, pred_boxes, pred_masks, config, outputs_class=None, outputs_coord=None, **kwargs +): + # First: create the matcher + matcher = HungarianMatcher(class_cost=config.class_cost, bbox_cost=config.bbox_cost, giou_cost=config.giou_cost) + # Second: create the criterion + losses = ["labels", "boxes", "cardinality", "masks"] + criterion = DeformableDetrImageLoss( + matcher=matcher, + num_classes=config.num_labels, + focal_alpha=config.focal_alpha, + losses=losses, + ) + criterion.to(device) + # Third: compute the losses, based on outputs and labels + outputs_loss = {} + outputs_loss["logits"] = logits + outputs_loss["pred_boxes"] = pred_boxes + outputs_loss["pred_masks"] = pred_masks + + auxiliary_outputs = None + if config.auxiliary_loss: + auxiliary_outputs = _set_aux_loss(outputs_class, outputs_coord) + outputs_loss["auxiliary_outputs"] = auxiliary_outputs + + loss_dict = criterion(outputs_loss, labels) + # Fourth: compute total loss, as a weighted sum of the various losses + weight_dict = {"loss_ce": 1, "loss_bbox": config.bbox_loss_coefficient} + weight_dict["loss_giou"] = config.giou_loss_coefficient + weight_dict["loss_mask"] = config.mask_loss_coefficient + weight_dict["loss_dice"] = config.dice_loss_coefficient + if config.auxiliary_loss: + aux_weight_dict = {} + for i in range(config.decoder_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + + loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) + return loss, loss_dict, auxiliary_outputs + + +def DeformableDetrForObjectDetectionLoss( + logits, labels, device, pred_boxes, config, outputs_class=None, outputs_coord=None, **kwargs +): + # First: create the matcher + matcher = DeformableDetrHungarianMatcher( + class_cost=config.class_cost, bbox_cost=config.bbox_cost, giou_cost=config.giou_cost + ) + # Second: create the criterion + losses = ["labels", "boxes", "cardinality"] + criterion = DeformableDetrImageLoss( + matcher=matcher, + num_classes=config.num_labels, + focal_alpha=config.focal_alpha, + losses=losses, + ) + criterion.to(device) + # Third: compute the losses, based on outputs and labels + outputs_loss = {} + auxiliary_outputs = None + outputs_loss["logits"] = logits + outputs_loss["pred_boxes"] = pred_boxes + if config.auxiliary_loss: + auxiliary_outputs = _set_aux_loss(outputs_class, outputs_coord) + outputs_loss["auxiliary_outputs"] = auxiliary_outputs + + loss_dict = criterion(outputs_loss, labels) + # Fourth: compute total loss, as a weighted sum of the various losses + weight_dict = {"loss_ce": 1, "loss_bbox": config.bbox_loss_coefficient} + weight_dict["loss_giou"] = config.giou_loss_coefficient + if config.auxiliary_loss: + aux_weight_dict = {} + for i in range(config.decoder_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) + return loss, loss_dict, auxiliary_outputs diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_for_object_detection.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_for_object_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..b820f6daed1224dace31fa30be24ac26094effd2 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_for_object_detection.py @@ -0,0 +1,562 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Optional + +import torch +import torch.nn as nn +from torch import Tensor + +from ..utils import is_accelerate_available, is_scipy_available, is_vision_available, requires_backends + + +if is_accelerate_available(): + from accelerate import PartialState + from accelerate.utils import reduce + +if is_scipy_available(): + from scipy.optimize import linear_sum_assignment + + +if is_vision_available(): + from transformers.image_transforms import center_to_corners_format + + +def dice_loss(inputs, targets, num_boxes): + """ + Compute the DICE loss, similar to generalized IOU for masks + + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs (0 for the negative class and 1 for the positive + class). + """ + inputs = inputs.sigmoid() + inputs = inputs.flatten(1) + numerator = 2 * (inputs * targets).sum(1) + denominator = inputs.sum(-1) + targets.sum(-1) + loss = 1 - (numerator + 1) / (denominator + 1) + return loss.sum() / num_boxes + + +def sigmoid_focal_loss(inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + + Args: + inputs (`torch.FloatTensor` of arbitrary shape): + The predictions for each example. + targets (`torch.FloatTensor` with the same shape as `inputs`) + A tensor storing the binary classification label for each element in the `inputs` (0 for the negative class + and 1 for the positive class). + alpha (`float`, *optional*, defaults to `0.25`): + Optional weighting factor in the range (0,1) to balance positive vs. negative examples. + gamma (`int`, *optional*, defaults to `2`): + Exponent of the modulating factor (1 - p_t) to balance easy vs hard examples. + + Returns: + Loss tensor + """ + prob = inputs.sigmoid() + ce_loss = nn.functional.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + # add modulating factor + p_t = prob * targets + (1 - prob) * (1 - targets) + loss = ce_loss * ((1 - p_t) ** gamma) + + if alpha >= 0: + alpha_t = alpha * targets + (1 - alpha) * (1 - targets) + loss = alpha_t * loss + + return loss.mean(1).sum() / num_boxes + + +# taken from https://github.com/facebookresearch/detr/blob/master/models/detr.py +class ImageLoss(nn.Module): + """ + This class computes the losses for DetrForObjectDetection/DetrForSegmentation. The process happens in two steps: 1) + we compute hungarian assignment between ground truth boxes and the outputs of the model 2) we supervise each pair + of matched ground-truth / prediction (supervise class and box). + + A note on the `num_classes` argument (copied from original repo in detr.py): "the naming of the `num_classes` + parameter of the criterion is somewhat misleading. It indeed corresponds to `max_obj_id` + 1, where `max_obj_id` is + the maximum id for a class in your dataset. For example, COCO has a `max_obj_id` of 90, so we pass `num_classes` to + be 91. As another example, for a dataset that has a single class with `id` 1, you should pass `num_classes` to be 2 + (`max_obj_id` + 1). For more details on this, check the following discussion + https://github.com/facebookresearch/detr/issues/108#issuecomment-650269223" + + + Args: + matcher (`DetrHungarianMatcher`): + Module able to compute a matching between targets and proposals. + num_classes (`int`): + Number of object categories, omitting the special no-object category. + eos_coef (`float`): + Relative classification weight applied to the no-object category. + losses (`List[str]`): + List of all the losses to be applied. See `get_loss` for a list of all available losses. + """ + + def __init__(self, matcher, num_classes, eos_coef, losses): + super().__init__() + self.matcher = matcher + self.num_classes = num_classes + self.eos_coef = eos_coef + self.losses = losses + empty_weight = torch.ones(self.num_classes + 1) + empty_weight[-1] = self.eos_coef + self.register_buffer("empty_weight", empty_weight) + + # removed logging parameter, which was part of the original implementation + def loss_labels(self, outputs, targets, indices, num_boxes): + """ + Classification loss (NLL) targets dicts must contain the key "class_labels" containing a tensor of dim + [nb_target_boxes] + """ + if "logits" not in outputs: + raise KeyError("No logits were found in the outputs") + source_logits = outputs["logits"] + + idx = self._get_source_permutation_idx(indices) + target_classes_o = torch.cat([t["class_labels"][J] for t, (_, J) in zip(targets, indices)]) + target_classes = torch.full( + source_logits.shape[:2], self.num_classes, dtype=torch.int64, device=source_logits.device + ) + target_classes[idx] = target_classes_o + + loss_ce = nn.functional.cross_entropy(source_logits.transpose(1, 2), target_classes, self.empty_weight) + losses = {"loss_ce": loss_ce} + + return losses + + @torch.no_grad() + def loss_cardinality(self, outputs, targets, indices, num_boxes): + """ + Compute the cardinality error, i.e. the absolute error in the number of predicted non-empty boxes. + + This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients. + """ + logits = outputs["logits"] + device = logits.device + target_lengths = torch.as_tensor([len(v["class_labels"]) for v in targets], device=device) + # Count the number of predictions that are NOT "no-object" (which is the last class) + card_pred = (logits.argmax(-1) != logits.shape[-1] - 1).sum(1) + card_err = nn.functional.l1_loss(card_pred.float(), target_lengths.float()) + losses = {"cardinality_error": card_err} + return losses + + def loss_boxes(self, outputs, targets, indices, num_boxes): + """ + Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss. + + Targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4]. The target boxes + are expected in format (center_x, center_y, w, h), normalized by the image size. + """ + if "pred_boxes" not in outputs: + raise KeyError("No predicted boxes found in outputs") + idx = self._get_source_permutation_idx(indices) + source_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + loss_bbox = nn.functional.l1_loss(source_boxes, target_boxes, reduction="none") + + losses = {} + losses["loss_bbox"] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag( + generalized_box_iou(center_to_corners_format(source_boxes), center_to_corners_format(target_boxes)) + ) + losses["loss_giou"] = loss_giou.sum() / num_boxes + return losses + + def loss_masks(self, outputs, targets, indices, num_boxes): + """ + Compute the losses related to the masks: the focal loss and the dice loss. + + Targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w]. + """ + if "pred_masks" not in outputs: + raise KeyError("No predicted masks found in outputs") + + source_idx = self._get_source_permutation_idx(indices) + target_idx = self._get_target_permutation_idx(indices) + source_masks = outputs["pred_masks"] + source_masks = source_masks[source_idx] + masks = [t["masks"] for t in targets] + # TODO use valid to mask invalid areas due to padding in loss + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + target_masks = target_masks.to(source_masks) + target_masks = target_masks[target_idx] + + # upsample predictions to the target size + source_masks = nn.functional.interpolate( + source_masks[:, None], size=target_masks.shape[-2:], mode="bilinear", align_corners=False + ) + source_masks = source_masks[:, 0].flatten(1) + + target_masks = target_masks.flatten(1) + target_masks = target_masks.view(source_masks.shape) + losses = { + "loss_mask": sigmoid_focal_loss(source_masks, target_masks, num_boxes), + "loss_dice": dice_loss(source_masks, target_masks, num_boxes), + } + return losses + + def _get_source_permutation_idx(self, indices): + # permute predictions following indices + batch_idx = torch.cat([torch.full_like(source, i) for i, (source, _) in enumerate(indices)]) + source_idx = torch.cat([source for (source, _) in indices]) + return batch_idx, source_idx + + def _get_target_permutation_idx(self, indices): + # permute targets following indices + batch_idx = torch.cat([torch.full_like(target, i) for i, (_, target) in enumerate(indices)]) + target_idx = torch.cat([target for (_, target) in indices]) + return batch_idx, target_idx + + def get_loss(self, loss, outputs, targets, indices, num_boxes): + loss_map = { + "labels": self.loss_labels, + "cardinality": self.loss_cardinality, + "boxes": self.loss_boxes, + "masks": self.loss_masks, + } + if loss not in loss_map: + raise ValueError(f"Loss {loss} not supported") + return loss_map[loss](outputs, targets, indices, num_boxes) + + def forward(self, outputs, targets): + """ + This performs the loss computation. + + Args: + outputs (`dict`, *optional*): + Dictionary of tensors, see the output specification of the model for the format. + targets (`List[dict]`, *optional*): + List of dicts, such that `len(targets) == batch_size`. The expected keys in each dict depends on the + losses applied, see each loss' doc. + """ + outputs_without_aux = {k: v for k, v in outputs.items() if k != "auxiliary_outputs"} + + # Retrieve the matching between the outputs of the last layer and the targets + indices = self.matcher(outputs_without_aux, targets) + + # Compute the average number of target boxes across all nodes, for normalization purposes + num_boxes = sum(len(t["class_labels"]) for t in targets) + num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device) + world_size = 1 + if is_accelerate_available(): + if PartialState._shared_state != {}: + num_boxes = reduce(num_boxes) + world_size = PartialState().num_processes + num_boxes = torch.clamp(num_boxes / world_size, min=1).item() + + # Compute all the requested losses + losses = {} + for loss in self.losses: + losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes)) + + # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. + if "auxiliary_outputs" in outputs: + for i, auxiliary_outputs in enumerate(outputs["auxiliary_outputs"]): + indices = self.matcher(auxiliary_outputs, targets) + for loss in self.losses: + if loss == "masks": + # Intermediate masks losses are too costly to compute, we ignore them. + continue + l_dict = self.get_loss(loss, auxiliary_outputs, targets, indices, num_boxes) + l_dict = {k + f"_{i}": v for k, v in l_dict.items()} + losses.update(l_dict) + + return losses + + +# taken from https://github.com/facebookresearch/detr/blob/master/models/matcher.py +class HungarianMatcher(nn.Module): + """ + This class computes an assignment between the targets and the predictions of the network. + + For efficiency reasons, the targets don't include the no_object. Because of this, in general, there are more + predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, while the others are + un-matched (and thus treated as non-objects). + + Args: + class_cost: + The relative weight of the classification error in the matching cost. + bbox_cost: + The relative weight of the L1 error of the bounding box coordinates in the matching cost. + giou_cost: + The relative weight of the giou loss of the bounding box in the matching cost. + """ + + def __init__(self, class_cost: float = 1, bbox_cost: float = 1, giou_cost: float = 1): + super().__init__() + requires_backends(self, ["scipy"]) + + self.class_cost = class_cost + self.bbox_cost = bbox_cost + self.giou_cost = giou_cost + if class_cost == 0 and bbox_cost == 0 and giou_cost == 0: + raise ValueError("All costs of the Matcher can't be 0") + + @torch.no_grad() + def forward(self, outputs, targets): + """ + Args: + outputs (`dict`): + A dictionary that contains at least these entries: + * "logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits + * "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates. + targets (`List[dict]`): + A list of targets (len(targets) = batch_size), where each target is a dict containing: + * "class_labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of + ground-truth + objects in the target) containing the class labels + * "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates. + + Returns: + `List[Tuple]`: A list of size `batch_size`, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + batch_size, num_queries = outputs["logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + out_prob = outputs["logits"].flatten(0, 1).softmax(-1) # [batch_size * num_queries, num_classes] + out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] + + # Also concat the target labels and boxes + target_ids = torch.cat([v["class_labels"] for v in targets]) + target_bbox = torch.cat([v["boxes"] for v in targets]) + + # Compute the classification cost. Contrary to the loss, we don't use the NLL, + # but approximate it in 1 - proba[target class]. + # The 1 is a constant that doesn't change the matching, it can be ommitted. + class_cost = -out_prob[:, target_ids] + + # Compute the L1 cost between boxes + bbox_cost = torch.cdist(out_bbox, target_bbox, p=1) + + # Compute the giou cost between boxes + giou_cost = -generalized_box_iou(center_to_corners_format(out_bbox), center_to_corners_format(target_bbox)) + + # Final cost matrix + cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost + cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() + + sizes = [len(v["boxes"]) for v in targets] + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))] + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +# below: bounding box utilities taken from https://github.com/facebookresearch/detr/blob/master/util/box_ops.py + + +def _upcast(t: Tensor) -> Tensor: + # Protects from numerical overflows in multiplications by upcasting to the equivalent higher type + if t.is_floating_point(): + return t if t.dtype in (torch.float32, torch.float64) else t.float() + else: + return t if t.dtype in (torch.int32, torch.int64) else t.int() + + +def box_area(boxes: Tensor) -> Tensor: + """ + Computes the area of a set of bounding boxes, which are specified by its (x1, y1, x2, y2) coordinates. + + Args: + boxes (`torch.FloatTensor` of shape `(number_of_boxes, 4)`): + Boxes for which the area will be computed. They are expected to be in (x1, y1, x2, y2) format with `0 <= x1 + < x2` and `0 <= y1 < y2`. + + Returns: + `torch.FloatTensor`: a tensor containing the area for each box. + """ + boxes = _upcast(boxes) + return (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + left_top = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + right_bottom = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + width_height = (right_bottom - left_top).clamp(min=0) # [N,M,2] + inter = width_height[:, :, 0] * width_height[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/. The boxes should be in [x0, y0, x1, y1] (corner) format. + + Returns: + `torch.FloatTensor`: a [N, M] pairwise matrix, where N = len(boxes1) and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + if not (boxes1[:, 2:] >= boxes1[:, :2]).all(): + raise ValueError(f"boxes1 must be in [x0, y0, x1, y1] (corner) format, but got {boxes1}") + if not (boxes2[:, 2:] >= boxes2[:, :2]).all(): + raise ValueError(f"boxes2 must be in [x0, y0, x1, y1] (corner) format, but got {boxes2}") + iou, union = box_iou(boxes1, boxes2) + + top_left = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + bottom_right = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + width_height = (bottom_right - top_left).clamp(min=0) # [N,M,2] + area = width_height[:, :, 0] * width_height[:, :, 1] + + return iou - (area - union) / area + + +# below: taken from https://github.com/facebookresearch/detr/blob/master/util/misc.py#L306 +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor: + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + if tensor_list[0].ndim == 3: + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + batch_shape = [len(tensor_list)] + max_size + batch_size, num_channels, height, width = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((batch_size, height, width), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("Only 3-dimensional tensors are supported") + return NestedTensor(tensor, mask) + + +# taken from https://github.com/facebookresearch/detr/blob/master/models/detr.py +@torch.jit.unused +def _set_aux_loss(outputs_class, outputs_coord): + # this is a workaround to make torchscript happy, as torchscript + # doesn't support dictionary with non-homogeneous values, such + # as a dict having both a Tensor and a list. + return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class[:-1], outputs_coord[:-1])] + + +def ForSegmentationLoss( + logits, labels, device, pred_boxes, pred_masks, config, outputs_class=None, outputs_coord=None, **kwargs +): + # First: create the matcher + matcher = HungarianMatcher(class_cost=config.class_cost, bbox_cost=config.bbox_cost, giou_cost=config.giou_cost) + # Second: create the criterion + losses = ["labels", "boxes", "cardinality", "masks"] + criterion = ImageLoss( + matcher=matcher, + num_classes=config.num_labels, + eos_coef=config.eos_coefficient, + losses=losses, + ) + criterion.to(device) + # Third: compute the losses, based on outputs and labels + outputs_loss = {} + outputs_loss["logits"] = logits + outputs_loss["pred_boxes"] = pred_boxes + outputs_loss["pred_masks"] = pred_masks + + auxiliary_outputs = None + if config.auxiliary_loss: + auxiliary_outputs = _set_aux_loss(outputs_class, outputs_coord) + outputs_loss["auxiliary_outputs"] = auxiliary_outputs + + loss_dict = criterion(outputs_loss, labels) + # Fourth: compute total loss, as a weighted sum of the various losses + weight_dict = {"loss_ce": 1, "loss_bbox": config.bbox_loss_coefficient} + weight_dict["loss_giou"] = config.giou_loss_coefficient + weight_dict["loss_mask"] = config.mask_loss_coefficient + weight_dict["loss_dice"] = config.dice_loss_coefficient + if config.auxiliary_loss: + aux_weight_dict = {} + for i in range(config.decoder_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) + return loss, loss_dict, auxiliary_outputs + + +def ForObjectDetectionLoss( + logits, labels, device, pred_boxes, config, outputs_class=None, outputs_coord=None, **kwargs +): + # First: create the matcher + matcher = HungarianMatcher(class_cost=config.class_cost, bbox_cost=config.bbox_cost, giou_cost=config.giou_cost) + # Second: create the criterion + losses = ["labels", "boxes", "cardinality"] + criterion = ImageLoss( + matcher=matcher, + num_classes=config.num_labels, + eos_coef=config.eos_coefficient, + losses=losses, + ) + criterion.to(device) + # Third: compute the losses, based on outputs and labels + outputs_loss = {} + auxiliary_outputs = None + outputs_loss["logits"] = logits + outputs_loss["pred_boxes"] = pred_boxes + if config.auxiliary_loss: + auxiliary_outputs = _set_aux_loss(outputs_class, outputs_coord) + outputs_loss["auxiliary_outputs"] = auxiliary_outputs + + loss_dict = criterion(outputs_loss, labels) + # Fourth: compute total loss, as a weighted sum of the various losses + weight_dict = {"loss_ce": 1, "loss_bbox": config.bbox_loss_coefficient} + weight_dict["loss_giou"] = config.giou_loss_coefficient + if config.auxiliary_loss: + aux_weight_dict = {} + for i in range(config.decoder_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) + return loss, loss_dict, auxiliary_outputs diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_grounding_dino.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_grounding_dino.py new file mode 100644 index 0000000000000000000000000000000000000000..0b5e4f6054953ac79161fb86453e0479a54332f1 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_grounding_dino.py @@ -0,0 +1,271 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn + +from ..image_transforms import center_to_corners_format +from ..utils import is_scipy_available +from .loss_for_object_detection import HungarianMatcher, ImageLoss, _set_aux_loss, generalized_box_iou + + +if is_scipy_available(): + from scipy.optimize import linear_sum_assignment + + +# Similar to the one used in `DeformableDetr` but we reduce with sum and normalize by num_boxes +# instead of mean. +def sigmoid_focal_loss( + inputs: torch.Tensor, + targets: torch.Tensor, + num_boxes: int, + alpha: float = 0.25, + gamma: float = 2, +): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + + Args: + inputs (`torch.FloatTensor` of arbitrary shape): + The predictions for each example. + targets (`torch.FloatTensor` with the same shape as `inputs`) + A tensor storing the binary classification label for each element in the `inputs` (0 for the negative class + and 1 for the positive class). + num_boxes (`int`): + The total number of boxes in the batch. + alpha (`float`, *optional*, defaults to 0.25): + Optional weighting factor in the range (0,1) to balance positive vs. negative examples. + gamma (`int`, *optional*, defaults to 2): + Exponent of the modulating factor (1 - p_t) to balance easy vs hard examples. + + Returns: + Loss tensor + """ + prob = inputs.sigmoid() + ce_loss = nn.functional.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + # add modulating factor + p_t = prob * targets + (1 - prob) * (1 - targets) + loss = ce_loss * ((1 - p_t) ** gamma) + + if alpha >= 0: + alpha_t = alpha * targets + (1 - alpha) * (1 - targets) + loss = alpha_t * loss + + return loss.sum() / num_boxes + + +class GroundingDinoHungarianMatcher(HungarianMatcher): + @torch.no_grad() + def forward(self, outputs, targets): + """ + Args: + outputs (`dict`): + A dictionary that contains at least these entries: + * "logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits + * "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates. + * "label_maps": Tuple of tensors of dim [num_classes, hidden_dim]. + targets (`List[dict]`): + A list of targets (len(targets) = batch_size), where each target is a dict containing: + * "class_labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of + ground-truth + objects in the target) containing the class labels + * "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates. + + Returns: + `List[Tuple]`: A list of size `batch_size`, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + batch_size, num_queries = outputs["logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + out_prob = outputs["logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, hidden_dim] + out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] + label_maps = outputs["label_maps"] + + # First take the label map for each class in each batch and then concatenate them + label_maps = torch.cat([label_map[target["class_labels"]] for label_map, target in zip(label_maps, targets)]) + # Normalize label maps based on number of tokens per class + label_maps = label_maps / label_maps.sum(dim=-1, keepdim=True) + + # Also concat the target labels and boxes + target_bbox = torch.cat([v["boxes"] for v in targets]) + + # Compute the classification cost. + alpha = 0.25 + gamma = 2.0 + neg_cost_class = (1 - alpha) * (out_prob**gamma) * (-(1 - out_prob + 1e-8).log()) + pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) + # Compute the classification cost by taking pos and neg cost in the appropriate index + class_cost = (pos_cost_class - neg_cost_class) @ label_maps.t() + + # Compute the L1 cost between boxes + bbox_cost = torch.cdist(out_bbox, target_bbox, p=1) + + # Compute the giou cost between boxes + giou_cost = -generalized_box_iou(center_to_corners_format(out_bbox), center_to_corners_format(target_bbox)) + + # Final cost matrix + cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost + cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() + + sizes = [len(v["boxes"]) for v in targets] + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))] + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +class GroundingDinoImageLoss(ImageLoss): + """ + This class computes the losses for `GroundingDinoForObjectDetection`. The process happens in two steps: 1) we + compute hungarian assignment between ground truth boxes and the outputs of the model 2) we supervise each pair of + matched ground-truth / prediction (supervise class and box). + + Args: + matcher (`GroundingDinoHungarianMatcher`): + Module able to compute a matching between targets and proposals. + focal_alpha (`float`): + Alpha parameter in focal loss. + losses (`List[str]`): + List of all the losses to be applied. See `get_loss` for a list of all available losses. + """ + + def __init__(self, matcher, focal_alpha, losses): + nn.Module.__init__(self) + self.matcher = matcher + self.focal_alpha = focal_alpha + self.losses = losses + + def _get_target_classes_one_hot(self, outputs, targets, indices): + """ + Create one_hot based on the matching indices + """ + logits = outputs["logits"] + # Add offsets to class_labels to select the correct label map + class_labels = torch.cat( + [ + target["class_labels"][J] + len(outputs["label_maps"][i]) if i > 0 else target["class_labels"][J] + for i, (target, (_, J)) in enumerate(zip(targets, indices)) + ] + ) + label_maps = torch.cat(outputs["label_maps"], dim=0) + + idx = self._get_source_permutation_idx(indices) + target_classes_onehot = torch.zeros_like(logits, device=logits.device, dtype=torch.long) + target_classes_onehot[idx] = label_maps[class_labels].to(torch.long) + + return target_classes_onehot + + def loss_labels(self, outputs, targets, indices, num_boxes): + """ + Classification loss (Binary focal loss) targets dicts must contain the key "class_labels" containing a tensor + of dim [nb_target_boxes] + """ + if "logits" not in outputs: + raise KeyError("No logits were found in the outputs") + if "text_mask" not in outputs: + raise KeyError("No text_mask were found in the outputs") + + target_classes_onehot = self._get_target_classes_one_hot(outputs, targets, indices) + source_logits = outputs["logits"] + text_mask = outputs["text_mask"] + + # Select only valid logits + source_logits = torch.masked_select(source_logits, text_mask) + target_classes_onehot = torch.masked_select(target_classes_onehot, text_mask) + + target_classes_onehot = target_classes_onehot.float() + loss_ce = sigmoid_focal_loss( + inputs=source_logits, + targets=target_classes_onehot, + num_boxes=num_boxes, + alpha=self.focal_alpha, + gamma=2, + ) + + losses = {"loss_ce": loss_ce} + + return losses + + +def GroundingDinoForObjectDetectionLoss( + logits, + labels, + device, + pred_boxes, + config, + label_maps, + text_mask, + outputs_class=None, + outputs_coord=None, + encoder_logits=None, + encoder_pred_boxes=None, +): + # First: create the matcher + matcher = GroundingDinoHungarianMatcher( + class_cost=config.class_cost, bbox_cost=config.bbox_cost, giou_cost=config.giou_cost + ) + # Second: create the criterion + losses = ["labels", "boxes", "cardinality"] + criterion = GroundingDinoImageLoss( + matcher=matcher, + focal_alpha=config.focal_alpha, + losses=losses, + ) + criterion.to(device) + # Third: compute the losses, based on outputs and labels + outputs_loss = {} + outputs_loss["logits"] = logits + outputs_loss["pred_boxes"] = pred_boxes + outputs_loss["label_maps"] = label_maps + outputs_loss["text_mask"] = text_mask + + auxiliary_outputs = None + if config.auxiliary_loss: + auxiliary_outputs = _set_aux_loss(outputs_class, outputs_coord) + for aux_output in auxiliary_outputs: + aux_output["label_maps"] = label_maps + aux_output["text_mask"] = text_mask + outputs_loss["auxiliary_outputs"] = auxiliary_outputs + + loss_dict = criterion(outputs_loss, labels) + + if config.two_stage: + encoder_outputs_loss = { + "logits": encoder_logits, + "pred_boxes": encoder_pred_boxes, + "label_maps": label_maps, + "text_mask": text_mask, + } + encoder_loss_dict = criterion(encoder_outputs_loss, labels) + encoder_loss_dict = {k + "_enc": v for k, v in encoder_loss_dict.items()} + loss_dict.update(encoder_loss_dict) + # Fourth: compute total loss, as a weighted sum of the various losses + weight_dict = { + "loss_ce": 2.0, + "loss_bbox": config.bbox_loss_coefficient, + "loss_giou": config.giou_loss_coefficient, + } + + if config.two_stage: + enc_weight_dict = {k + "_enc": v for k, v in weight_dict.items()} + weight_dict.update(enc_weight_dict) + + if config.auxiliary_loss: + aux_weight_dict = {} + for i in range(config.decoder_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + + loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) + return loss, loss_dict, auxiliary_outputs diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_rt_detr.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_rt_detr.py new file mode 100644 index 0000000000000000000000000000000000000000..88a4ac7cf4fa8ac13e5f96e75e757c3ec4f666c1 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_rt_detr.py @@ -0,0 +1,471 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import is_scipy_available, is_vision_available, requires_backends +from .loss_for_object_detection import ( + box_iou, + dice_loss, + generalized_box_iou, + nested_tensor_from_tensor_list, + sigmoid_focal_loss, +) + + +if is_scipy_available(): + from scipy.optimize import linear_sum_assignment + + +if is_vision_available(): + from transformers.image_transforms import center_to_corners_format + + +# different for RT-DETR: not slicing the last element like in DETR one +@torch.jit.unused +def _set_aux_loss(outputs_class, outputs_coord): + # this is a workaround to make torchscript happy, as torchscript + # doesn't support dictionary with non-homogeneous values, such + # as a dict having both a Tensor and a list. + return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class, outputs_coord)] + + +class RTDetrHungarianMatcher(nn.Module): + """This class computes an assignment between the targets and the predictions of the network + + For efficiency reasons, the targets don't include the no_object. Because of this, in general, there are more + predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, while the others are + un-matched (and thus treated as non-objects). + + Args: + config: RTDetrConfig + """ + + def __init__(self, config): + super().__init__() + requires_backends(self, ["scipy"]) + + self.class_cost = config.matcher_class_cost + self.bbox_cost = config.matcher_bbox_cost + self.giou_cost = config.matcher_giou_cost + + self.use_focal_loss = config.use_focal_loss + self.alpha = config.matcher_alpha + self.gamma = config.matcher_gamma + + if self.class_cost == self.bbox_cost == self.giou_cost == 0: + raise ValueError("All costs of the Matcher can't be 0") + + @torch.no_grad() + def forward(self, outputs, targets): + """Performs the matching + + Params: + outputs: This is a dict that contains at least these entries: + "logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits + "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates + + targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: + "class_labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth + objects in the target) containing the class labels + "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates + + Returns: + A list of size batch_size, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + batch_size, num_queries = outputs["logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] + # Also concat the target labels and boxes + target_ids = torch.cat([v["class_labels"] for v in targets]) + target_bbox = torch.cat([v["boxes"] for v in targets]) + # Compute the classification cost. Contrary to the loss, we don't use the NLL, + # but approximate it in 1 - proba[target class]. + # The 1 is a constant that doesn't change the matching, it can be ommitted. + if self.use_focal_loss: + out_prob = F.sigmoid(outputs["logits"].flatten(0, 1)) + out_prob = out_prob[:, target_ids] + neg_cost_class = (1 - self.alpha) * (out_prob**self.gamma) * (-(1 - out_prob + 1e-8).log()) + pos_cost_class = self.alpha * ((1 - out_prob) ** self.gamma) * (-(out_prob + 1e-8).log()) + class_cost = pos_cost_class - neg_cost_class + else: + out_prob = outputs["logits"].flatten(0, 1).softmax(-1) # [batch_size * num_queries, num_classes] + class_cost = -out_prob[:, target_ids] + + # Compute the L1 cost between boxes + bbox_cost = torch.cdist(out_bbox, target_bbox, p=1) + # Compute the giou cost betwen boxes + giou_cost = -generalized_box_iou(center_to_corners_format(out_bbox), center_to_corners_format(target_bbox)) + # Compute the final cost matrix + cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost + cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() + + sizes = [len(v["boxes"]) for v in targets] + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(cost_matrix.split(sizes, -1))] + + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +class RTDetrLoss(nn.Module): + """ + This class computes the losses for RTDetr. The process happens in two steps: 1) we compute hungarian assignment + between ground truth boxes and the outputs of the model 2) we supervise each pair of matched ground-truth / + prediction (supervise class and box). + + Args: + matcher (`DetrHungarianMatcher`): + Module able to compute a matching between targets and proposals. + weight_dict (`Dict`): + Dictionary relating each loss with its weights. These losses are configured in RTDetrConf as + `weight_loss_vfl`, `weight_loss_bbox`, `weight_loss_giou` + losses (`List[str]`): + List of all the losses to be applied. See `get_loss` for a list of all available losses. + alpha (`float`): + Parameter alpha used to compute the focal loss. + gamma (`float`): + Parameter gamma used to compute the focal loss. + eos_coef (`float`): + Relative classification weight applied to the no-object category. + num_classes (`int`): + Number of object categories, omitting the special no-object category. + """ + + def __init__(self, config): + super().__init__() + + self.matcher = RTDetrHungarianMatcher(config) + self.num_classes = config.num_labels + self.weight_dict = { + "loss_vfl": config.weight_loss_vfl, + "loss_bbox": config.weight_loss_bbox, + "loss_giou": config.weight_loss_giou, + } + self.losses = ["vfl", "boxes"] + self.eos_coef = config.eos_coefficient + empty_weight = torch.ones(config.num_labels + 1) + empty_weight[-1] = self.eos_coef + self.register_buffer("empty_weight", empty_weight) + self.alpha = config.focal_loss_alpha + self.gamma = config.focal_loss_gamma + + def loss_labels_vfl(self, outputs, targets, indices, num_boxes, log=True): + if "pred_boxes" not in outputs: + raise KeyError("No predicted boxes found in outputs") + if "logits" not in outputs: + raise KeyError("No predicted logits found in outputs") + idx = self._get_source_permutation_idx(indices) + + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([_target["boxes"][i] for _target, (_, i) in zip(targets, indices)], dim=0) + ious, _ = box_iou(center_to_corners_format(src_boxes.detach()), center_to_corners_format(target_boxes)) + ious = torch.diag(ious) + + src_logits = outputs["logits"] + target_classes_original = torch.cat([_target["class_labels"][i] for _target, (_, i) in zip(targets, indices)]) + target_classes = torch.full( + src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device + ) + target_classes[idx] = target_classes_original + target = F.one_hot(target_classes, num_classes=self.num_classes + 1)[..., :-1] + + target_score_original = torch.zeros_like(target_classes, dtype=src_logits.dtype) + target_score_original[idx] = ious.to(target_score_original.dtype) + target_score = target_score_original.unsqueeze(-1) * target + + pred_score = F.sigmoid(src_logits.detach()) + weight = self.alpha * pred_score.pow(self.gamma) * (1 - target) + target_score + + loss = F.binary_cross_entropy_with_logits(src_logits, target_score, weight=weight, reduction="none") + loss = loss.mean(1).sum() * src_logits.shape[1] / num_boxes + return {"loss_vfl": loss} + + def loss_labels(self, outputs, targets, indices, num_boxes, log=True): + """Classification loss (NLL) + targets dicts must contain the key "class_labels" containing a tensor of dim [nb_target_boxes] + """ + if "logits" not in outputs: + raise KeyError("No logits were found in the outputs") + + src_logits = outputs["logits"] + + idx = self._get_source_permutation_idx(indices) + target_classes_original = torch.cat([_target["class_labels"][i] for _target, (_, i) in zip(targets, indices)]) + target_classes = torch.full( + src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device + ) + target_classes[idx] = target_classes_original + + loss_ce = F.cross_entropy(src_logits.transpose(1, 2), target_classes, self.class_weight) + losses = {"loss_ce": loss_ce} + return losses + + @torch.no_grad() + def loss_cardinality(self, outputs, targets, indices, num_boxes): + """ + Compute the cardinality error, i.e. the absolute error in the number of predicted non-empty boxes. This is not + really a loss, it is intended for logging purposes only. It doesn't propagate gradients. + """ + logits = outputs["logits"] + device = logits.device + target_lengths = torch.as_tensor([len(v["class_labels"]) for v in targets], device=device) + # Count the number of predictions that are NOT "no-object" (which is the last class) + card_pred = (logits.argmax(-1) != logits.shape[-1] - 1).sum(1) + card_err = nn.functional.l1_loss(card_pred.float(), target_lengths.float()) + losses = {"cardinality_error": card_err} + return losses + + def loss_boxes(self, outputs, targets, indices, num_boxes): + """ + Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss. Targets dicts must + contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4]. The target boxes are expected in + format (center_x, center_y, w, h), normalized by the image size. + """ + if "pred_boxes" not in outputs: + raise KeyError("No predicted boxes found in outputs") + idx = self._get_source_permutation_idx(indices) + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + losses = {} + + loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction="none") + losses["loss_bbox"] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag( + generalized_box_iou(center_to_corners_format(src_boxes), center_to_corners_format(target_boxes)) + ) + losses["loss_giou"] = loss_giou.sum() / num_boxes + return losses + + def loss_masks(self, outputs, targets, indices, num_boxes): + """ + Compute the losses related to the masks: the focal loss and the dice loss. Targets dicts must contain the key + "masks" containing a tensor of dim [nb_target_boxes, h, w]. + """ + if "pred_masks" not in outputs: + raise KeyError("No predicted masks found in outputs") + + source_idx = self._get_source_permutation_idx(indices) + target_idx = self._get_target_permutation_idx(indices) + source_masks = outputs["pred_masks"] + source_masks = source_masks[source_idx] + masks = [t["masks"] for t in targets] + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + target_masks = target_masks.to(source_masks) + target_masks = target_masks[target_idx] + + # upsample predictions to the target size + source_masks = nn.functional.interpolate( + source_masks[:, None], size=target_masks.shape[-2:], mode="bilinear", align_corners=False + ) + source_masks = source_masks[:, 0].flatten(1) + + target_masks = target_masks.flatten(1) + target_masks = target_masks.view(source_masks.shape) + losses = { + "loss_mask": sigmoid_focal_loss(source_masks, target_masks, num_boxes), + "loss_dice": dice_loss(source_masks, target_masks, num_boxes), + } + return losses + + def loss_labels_bce(self, outputs, targets, indices, num_boxes, log=True): + src_logits = outputs["logits"] + idx = self._get_source_permutation_idx(indices) + target_classes_original = torch.cat([_target["class_labels"][i] for _target, (_, i) in zip(targets, indices)]) + target_classes = torch.full( + src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device + ) + target_classes[idx] = target_classes_original + + target = F.one_hot(target_classes, num_classes=self.num_classes + 1)[..., :-1] + loss = F.binary_cross_entropy_with_logits(src_logits, target * 1.0, reduction="none") + loss = loss.mean(1).sum() * src_logits.shape[1] / num_boxes + return {"loss_bce": loss} + + def _get_source_permutation_idx(self, indices): + # permute predictions following indices + batch_idx = torch.cat([torch.full_like(source, i) for i, (source, _) in enumerate(indices)]) + source_idx = torch.cat([source for (source, _) in indices]) + return batch_idx, source_idx + + def _get_target_permutation_idx(self, indices): + # permute targets following indices + batch_idx = torch.cat([torch.full_like(target, i) for i, (_, target) in enumerate(indices)]) + target_idx = torch.cat([target for (_, target) in indices]) + return batch_idx, target_idx + + def loss_labels_focal(self, outputs, targets, indices, num_boxes, log=True): + if "logits" not in outputs: + raise KeyError("No logits found in outputs") + + src_logits = outputs["logits"] + + idx = self._get_source_permutation_idx(indices) + target_classes_original = torch.cat([_target["class_labels"][i] for _target, (_, i) in zip(targets, indices)]) + target_classes = torch.full( + src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device + ) + target_classes[idx] = target_classes_original + + target = F.one_hot(target_classes, num_classes=self.num_classes + 1)[..., :-1] + loss = sigmoid_focal_loss(src_logits, target, self.alpha, self.gamma) + loss = loss.mean(1).sum() * src_logits.shape[1] / num_boxes + return {"loss_focal": loss} + + def get_loss(self, loss, outputs, targets, indices, num_boxes): + loss_map = { + "labels": self.loss_labels, + "cardinality": self.loss_cardinality, + "boxes": self.loss_boxes, + "masks": self.loss_masks, + "bce": self.loss_labels_bce, + "focal": self.loss_labels_focal, + "vfl": self.loss_labels_vfl, + } + if loss not in loss_map: + raise ValueError(f"Loss {loss} not supported") + return loss_map[loss](outputs, targets, indices, num_boxes) + + @staticmethod + def get_cdn_matched_indices(dn_meta, targets): + dn_positive_idx, dn_num_group = dn_meta["dn_positive_idx"], dn_meta["dn_num_group"] + num_gts = [len(t["class_labels"]) for t in targets] + device = targets[0]["class_labels"].device + + dn_match_indices = [] + for i, num_gt in enumerate(num_gts): + if num_gt > 0: + gt_idx = torch.arange(num_gt, dtype=torch.int64, device=device) + gt_idx = gt_idx.tile(dn_num_group) + assert len(dn_positive_idx[i]) == len(gt_idx) + dn_match_indices.append((dn_positive_idx[i], gt_idx)) + else: + dn_match_indices.append( + ( + torch.zeros(0, dtype=torch.int64, device=device), + torch.zeros(0, dtype=torch.int64, device=device), + ) + ) + + return dn_match_indices + + def forward(self, outputs, targets): + """ + This performs the loss computation. + + Args: + outputs (`dict`, *optional*): + Dictionary of tensors, see the output specification of the model for the format. + targets (`List[dict]`, *optional*): + List of dicts, such that `len(targets) == batch_size`. The expected keys in each dict depends on the + losses applied, see each loss' doc. + """ + outputs_without_aux = {k: v for k, v in outputs.items() if "auxiliary_outputs" not in k} + + # Retrieve the matching between the outputs of the last layer and the targets + indices = self.matcher(outputs_without_aux, targets) + + # Compute the average number of target boxes across all nodes, for normalization purposes + num_boxes = sum(len(t["class_labels"]) for t in targets) + num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device) + num_boxes = torch.clamp(num_boxes, min=1).item() + + # Compute all the requested losses + losses = {} + for loss in self.losses: + l_dict = self.get_loss(loss, outputs, targets, indices, num_boxes) + l_dict = {k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict} + losses.update(l_dict) + + # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. + if "auxiliary_outputs" in outputs: + for i, auxiliary_outputs in enumerate(outputs["auxiliary_outputs"]): + indices = self.matcher(auxiliary_outputs, targets) + for loss in self.losses: + if loss == "masks": + # Intermediate masks losses are too costly to compute, we ignore them. + continue + l_dict = self.get_loss(loss, auxiliary_outputs, targets, indices, num_boxes) + l_dict = {k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict} + l_dict = {k + f"_aux_{i}": v for k, v in l_dict.items()} + losses.update(l_dict) + + # In case of cdn auxiliary losses. For rtdetr + if "dn_auxiliary_outputs" in outputs: + if "denoising_meta_values" not in outputs: + raise ValueError( + "The output must have the 'denoising_meta_values` key. Please, ensure that 'outputs' includes a 'denoising_meta_values' entry." + ) + indices = self.get_cdn_matched_indices(outputs["denoising_meta_values"], targets) + num_boxes = num_boxes * outputs["denoising_meta_values"]["dn_num_group"] + + for i, auxiliary_outputs in enumerate(outputs["dn_auxiliary_outputs"]): + # indices = self.matcher(auxiliary_outputs, targets) + for loss in self.losses: + if loss == "masks": + # Intermediate masks losses are too costly to compute, we ignore them. + continue + kwargs = {} + l_dict = self.get_loss(loss, auxiliary_outputs, targets, indices, num_boxes, **kwargs) + l_dict = {k: l_dict[k] * self.weight_dict[k] for k in l_dict if k in self.weight_dict} + l_dict = {k + f"_dn_{i}": v for k, v in l_dict.items()} + losses.update(l_dict) + + return losses + + +def RTDetrForObjectDetectionLoss( + logits, + labels, + device, + pred_boxes, + config, + outputs_class=None, + outputs_coord=None, + enc_topk_logits=None, + enc_topk_bboxes=None, + denoising_meta_values=None, + **kwargs, +): + criterion = RTDetrLoss(config) + criterion.to(device) + # Second: compute the losses, based on outputs and labels + outputs_loss = {} + outputs_loss["logits"] = logits + outputs_loss["pred_boxes"] = pred_boxes + if config.auxiliary_loss: + if denoising_meta_values is not None: + dn_out_coord, outputs_coord = torch.split(outputs_coord, denoising_meta_values["dn_num_split"], dim=2) + dn_out_class, outputs_class = torch.split(outputs_class, denoising_meta_values["dn_num_split"], dim=2) + + auxiliary_outputs = _set_aux_loss(outputs_class[:, :-1].transpose(0, 1), outputs_coord[:, :-1].transpose(0, 1)) + outputs_loss["auxiliary_outputs"] = auxiliary_outputs + outputs_loss["auxiliary_outputs"].extend(_set_aux_loss([enc_topk_logits], [enc_topk_bboxes])) + if denoising_meta_values is not None: + outputs_loss["dn_auxiliary_outputs"] = _set_aux_loss( + dn_out_class.transpose(0, 1), dn_out_coord.transpose(0, 1) + ) + outputs_loss["denoising_meta_values"] = denoising_meta_values + + loss_dict = criterion(outputs_loss, labels) + + loss = sum(loss_dict.values()) + return loss, loss_dict, auxiliary_outputs diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_utils.py b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0e052aed6a9b6f73266f4cbce4dc8cde38ea9481 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/loss_utils.py @@ -0,0 +1,158 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +import torch.nn as nn +from torch.nn import BCEWithLogitsLoss, MSELoss + +from .loss_deformable_detr import DeformableDetrForObjectDetectionLoss, DeformableDetrForSegmentationLoss +from .loss_for_object_detection import ForObjectDetectionLoss, ForSegmentationLoss +from .loss_grounding_dino import GroundingDinoForObjectDetectionLoss +from .loss_rt_detr import RTDetrForObjectDetectionLoss + + +def fixed_cross_entropy( + source: torch.Tensor, + target: torch.Tensor, + num_items_in_batch: Optional[int] = None, + ignore_index: int = -100, + **kwargs, +) -> torch.Tensor: + reduction = "sum" if num_items_in_batch is not None else "mean" + loss = nn.functional.cross_entropy(source, target, ignore_index=ignore_index, reduction=reduction) + if reduction == "sum": + loss = loss / num_items_in_batch + return loss + + +def ForCausalLMLoss( + logits, + labels, + vocab_size: int, + num_items_in_batch: Optional[int] = None, + ignore_index: int = -100, + shift_labels: Optional[torch.Tensor] = None, + **kwargs, +) -> torch.Tensor: + # Upcast to float if we need to compute the loss to avoid potential precision issues + logits = logits.float() + + if shift_labels is None: + # Shift so that tokens < n predict n + labels = nn.functional.pad(labels, (0, 1), value=ignore_index) + shift_labels = labels[..., 1:].contiguous() + + # Flatten the tokens + logits = logits.view(-1, vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(logits.device) + loss = fixed_cross_entropy(logits, shift_labels, num_items_in_batch, ignore_index, **kwargs) + return loss + + +def ForMaskedLMLoss( + logits: torch.Tensor, + labels: torch.Tensor, + vocab_size: int, + num_items_in_batch: Optional[int] = None, + ignore_index: int = -100, + **kwargs, +): + # Upcast to float if we need to compute the loss to avoid potential precision issues + logits = logits.float() + + # Flatten the tokens + logits = logits.view(-1, vocab_size) + labels = labels.view(-1) + # Enable model parallelism + + labels = labels.to(logits.device) + loss = fixed_cross_entropy(logits, labels, num_items_in_batch, ignore_index, **kwargs) + return loss + + +def ForSequenceClassificationLoss(labels: torch.Tensor, pooled_logits: torch.Tensor, config, **kwargs) -> torch.Tensor: + num_labels = config.num_labels + if config.problem_type is None: + if num_labels == 1: + config.problem_type = "regression" + elif num_labels > 1 and (labels.dtype in (torch.long, torch.int)): + config.problem_type = "single_label_classification" + else: + config.problem_type = "multi_label_classification" + + labels = labels.to(pooled_logits.device) + if config.problem_type == "regression": + loss_fct = MSELoss() + if num_labels == 1: + return loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + return loss_fct(pooled_logits, labels) + if config.problem_type == "single_label_classification": + return fixed_cross_entropy(pooled_logits.view(-1, num_labels), labels.view(-1), **kwargs) + + if config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + return loss_fct(pooled_logits, labels) + + raise RuntimeError(f"Invalid problem type: {config.problem_type}") + + +def ForQuestionAnsweringLoss(start_logits, end_logits, start_positions, end_positions, **kwargs): + total_loss = None + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1).to(start_logits.device) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1).to(end_logits.device) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions = start_positions.clamp(0, ignored_index) + end_positions = end_positions.clamp(0, ignored_index) + + start_loss = fixed_cross_entropy(start_logits, start_positions, ignore_index=ignored_index, **kwargs) + end_loss = fixed_cross_entropy(end_logits, end_positions, ignore_index=ignored_index, **kwargs) + total_loss = (start_loss + end_loss) / 2 + return total_loss + + +def ForTokenClassification(logits: torch.Tensor, labels, config, **kwargs): + # Upcast to float if we need to compute the loss to avoid potential precision issues + logits = logits.view(-1, config.num_labels) + labels = labels.view(-1).to(logits.device) + logits = logits.float() + # Flatten the tokens + return fixed_cross_entropy(logits, labels, **kwargs) + + +LOSS_MAPPING = { + "ForCausalLM": ForCausalLMLoss, + "ForMaskedLM": ForMaskedLMLoss, + "ForQuestionAnswering": ForQuestionAnsweringLoss, + "ForSequenceClassification": ForSequenceClassificationLoss, + "ForTokenClassification": ForTokenClassification, + "ForSegmentation": ForSegmentationLoss, + "ForObjectDetection": ForObjectDetectionLoss, + "DeformableDetrForObjectDetection": DeformableDetrForObjectDetectionLoss, + "ConditionalDetrForObjectDetection": DeformableDetrForObjectDetectionLoss, + "DabDetrForObjectDetection": DeformableDetrForObjectDetectionLoss, + "GroundingDinoForObjectDetection": GroundingDinoForObjectDetectionLoss, + "ConditionalDetrForSegmentation": DeformableDetrForSegmentationLoss, + "RTDetrForObjectDetection": RTDetrForObjectDetectionLoss, + "RTDetrV2ForObjectDetection": RTDetrForObjectDetectionLoss, +} diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/merges.txt b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..31349551d90c7606f325fe0f11bbb8bd5fa0d7c7 --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/merges.txt @@ -0,0 +1,151388 @@ +#version: 0.2 +Ġ Ġ +ĠĠ ĠĠ +i n +Ġ t +ĠĠĠĠ ĠĠĠĠ +e r +ĠĠ Ġ +o n +Ġ a +r e +a t +s t +e n +o r +Ġt h +Ċ Ċ +Ġ c +l e +Ġ s +i t +a n +a r +a l +Ġth e +; Ċ +Ġ p +Ġ f +o u +Ġ = +i s +ĠĠĠĠ ĠĠĠ +in g +e s +Ġ w +i on +e d +i c +Ġ b +Ġ d +e t +Ġ m +Ġ o +ĉ ĉ +r o +a s +e l +c t +n d +Ġ in +Ġ h +en t +i d +Ġ n +a m +ĠĠĠĠĠĠĠĠ ĠĠĠ +Ġt o +Ġ re +- - +Ġ { +Ġo f +o m +) ;Ċ +i m +č Ċ +Ġ ( +i l +/ / +Ġa nd +u r +s e +Ġ l +e x +Ġ S +a d +Ġ " +c h +u t +i f +* * +Ġ } +e m +o l +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ +t h +) Ċ +Ġ{ Ċ +Ġ g +i g +i v +, Ċ +c e +o d +Ġ v +at e +Ġ T +a g +a y +Ġ * +o t +u s +Ġ C +Ġ st +Ġ I +u n +u l +u e +Ġ A +o w +Ġ ' +e w +Ġ < +at ion +( ) +Ġf or +a b +or t +u m +am e +Ġ is +p e +t r +c k +â Ģ +Ġ y +i st +-- -- +. ĊĊ +h e +Ġ e +l o +Ġ M +Ġb e +er s +Ġ on +Ġc on +a p +u b +Ġ P +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ +as s +in t +> Ċ +l y +ur n +Ġ $ +; ĊĊ +a v +p ort +i r +- > +n t +ct ion +en d +Ġd e +it h +ou t +t urn +ou r +ĠĠĠĠ Ġ +l ic +re s +p t += = +Ġth is +Ġw h +Ġ if +Ġ D +v er +ag e +Ġ B +h t +ex t += " +Ġth at +** ** +Ġ R +Ġ it +es s +Ġ F +Ġ r +o s +an d +Ġa s +e ct +k e +ro m +Ġ // +c on +Ġ L +( " +q u +l ass +Ġw ith +i z +d e +Ġ N +Ġa l +o p +u p +g et +Ġ} Ċ +i le +Ġa n +at a +o re +r i +Ġp ro +; čĊ +ĉĉ ĉĉ +t er +a in +Ġ W +Ġ E +Ġc om +Ġre turn +ar t +Ġ H +a ck +im port +ub lic +Ġ or +e st +m ent +Ġ G +ab le +Ġ - +in e +il l +in d +er e +: : +it y +Ġ + +Ġt r +el f +ig ht +( ' +or m +ul t +st r +. . +" , +Ġy ou +y pe +p l +Ġn ew +Ġ j +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +Ġf rom +Ġ ex +Ġ O +l d +Ġ [ +o c +: Ċ +Ġs e +Ġ le +---- ---- +. s +{ Ċ +' , +an t +Ġa t +as e +. c +Ġc h +< / +av e +an g +Ġa re +Ġin t +âĢ Ļ +_ t +er t +i al +a ct +} Ċ +iv e +od e +o st +Ġc lass +Ġn ot +o g +or d +al ue +al l +f f +( );Ċ +on t +im e +a re +Ġ U +Ġp r +Ġ : +i es +iz e +u re +Ġb y +i re +Ġ} ĊĊ +. p +Ġs h +ic e +a st +pt ion +tr ing +o k +_ _ +c l +# # +Ġh e +ar d +) . +Ġ @ +i ew +ĉĉ ĉ +Ġw as +i p +th is +Ġ u +ĠT he +id e +a ce +i b +a c +r ou +Ġw e +j ect +Ġp ublic +a k +v e +at h +o id +Ġ= > +u st +q ue +Ġre s +) ) +' s +Ġ k +an s +y st +un ction +**** **** +Ġ i +Ġ us +p p +on e +a il +== == +n ame +Ġst r +Ġ / +Ġ & +a ch +d iv +yst em +el l +Ġh ave +er r +ou ld +ul l +p on +Ġ J +_ p +Ġ= = +ig n +S t +. Ċ +Ġp l +) ;ĊĊ +f orm +p ut +ou nt +} ĊĊ +d d +it e +Ġg et +r r +om e +Ġ âĢ +ar am +c c +Ġ* / +E R +I n +le s +_ s +on g +i e +Ġc an +Ġ V +er v +p r +Ġ un +ro w +b er +Ġd o +l l +Ġ el +Ġs elf +at ed +ar y +Ġ . +' ] +u d +Ġ en +ĠT h +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ +t e +_ c +u ct +Ġa b +or k +. get +Ġ # +a w +res s +o b +N ame +ap p +[ ' +Ġal l +or y +it ion +an ce +e ar +Ġcon t +v ent +i a +Ġw ill +I N +ĠĠĠĠĠĠĠĠ Ġ +re turn +Ġ< / +d ata +) ĊĊ +R e +p le +il d +th er +Ġy our +" Ċ +( $ +Ġ out +) , +Ġh as +S tring +s o +Ġ up +a x +Ġde f +Ġb o +g e +al se +O N +p er +ic h +Ġb ut +Ġ Ċ +Ġ _ +_ m +ad d +que st +od el +s elf +er y +f t +en s +// // +a ke +. C +Ġg o +Ġf unction +Ġ K +iv ate +Ġ im +Ġcon st +. t +Ġ*/ Ċ +) ;čĊ +Ġv oid +Ġs et +ĠS ystem +c ri +( )Ċ +l i +ĉ if +. m +al ly +s et +e p +âĢĻ s +b o +de f +' ,Ċ +Ġm e +Ġ ! +at ch +" > +" ,Ċ +e c +ĠI n +p h +Ġ | +_ f +Ġv ar +en ce +I d +re e +in k +le ct +u g +et h +Ġel se +-------- -------- +con t +Ġs o +at ic +Ġl o +p ro +t on +s s +ow n +ab el +o int +ou s +el d +S T +T he +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +R E +" : +ol or +t p +e g +ke y +u de +ĠS t +ou nd +Ġa r +" );Ċ +en er +s er +b ject +ess age +f er +Ġm ore +ation s +ent s +Ġh is +Ġthe y +. S +Ġ Y +u se +n e +is h +ol d +_ d +i o +i eld +Ġp er +C ont +ing s +## ## +Ġd ata +Ġs a +e f +f o +Ġon e +en g +Ġd is +A T +Ġn ame +Ġtr ue +v al +le d +. f +Ġn e +Ġ end +. T +c re +ar k +lo g +E x +err or +_ id +ur re +ang e +Ġn ull +rr ay +Ġm y +p an +ic t +at or +V iew +L ist +ĉ return +âĢ Ŀ +Ġp re +Ġ x +cl ude +ar g +o v +. h +Ġ > +Ġthe ir +' ) +ir st +ic k +g h +L E +O R +Ġpr ivate +t em +čĊ čĊ +us er +Ġ ) +c om +. A +" ;Ċ +Ġ id +re ad +Ġwh o +_ b +" >Ċ +Ġt ime +Ġm an +r y +==== ==== +rou p +ro p +p ublic +v el +um ber +b le +Ġwh ich +******** ******** +Ġan y +Ġf alse +w e +Ġv alue +Ġl i +" ) +nd er +g r +Ġn o +p aram +f ig +.c om +Ġa pp +_ l +ion s +. D +ĠC h +Ġab out +Ġa dd +Ġs u +Ġstr ing +I D +Ġo ver +str ing +. l +our ce +_ C +] Ċ +Ġ qu +ĠS tring +c a +S E +Ġ ro +s h +u al +T ype +s on +n ew +er n +Ġa g +A R +] ;Ċ +] . +Ġ ? +ic al +Ġd es +ut h +i x +ay s +Ġt ype +' t +a ult +Ġin ter +v ar +. b +Ġp art +. d +urre nt +I T +E N +en c +( f +r a +v alue +ch o +ut ton +o se +Ġ! = +at er +à © +re ate +ol l +p os +y le +n g +A L +us ing +am es +Ġ{ čĊ +at es +el y +Ġw ork +Ġ em +in al +Ġs p +Ġwh en +.s et +ĠĠĠĠ ĠĠ +) :Ċ +t o +qu ire +ind ow +le ment +pe ct +as h +[ i +Ġu se +. F +pe c +Ġa d +o ve +ce ption +eng th +in clude +ad er +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +at us +T h +it le +r it +v oid +() . +( Ċ +Ġof f +Ġo ther +Ġ& & +' ;Ċ +m s +Ġbe en +Ġt e +m l +c o +n c +erv ice +Ġ % +** Ċ +an n +ad e +ĊĊ ĊĊ +lo ck +con st +pon se +Ġs up ++ + +d ate +Ġa cc +Ġh ad +Ġb u +ĠR e +Ġw ere +Ġf ile +Ġw ould +ĠâĢ ľ +v en +is s +Ġ our +c lass +r aw +Ġy ear +D ata +Ġv al +Ġs ome +f ter +y s +Ġ// / +rou nd +v iew +Ġp e +Ġth ere +Ġsa id +d u +o f +l ine +/ * +d uct +Ġh er +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +R es +Ġc o +Ġcom m +is e +m in +ĠĠĠĠ Ċ +# include +eth od +. P +ut e +Ġas s +I nt +as k +lo c +Ġli ke +od y +Ġle t +lo ad +Ġa m +ro l +Ġg r +y p +Ġal so +ĠI t +ur l +if ic +or s +_ P +_ n +ig h +Ġth an +C om +A N +U L +at ing +ĠTh is +re f +_ S +Ġst atic +ro ll +Ġj ust +Ġres ult +i an +id th +Ġthe m +) );Ċ +d er +re ak +C on +: // +u le +.. . +ar ch +em ent +Ġ< < +us h +en se +ar r +Ġint o +c ess +am p +i ed +um ent +Ġ \ +] , +w o +al s +Ġwh at +an c +V alue += ' +ol um +Ġp os +ag es +ay er +Ġs c +u es +" )Ċ +_ T +Ġl ist +( s +Ġc ase +C h +ĉĉĉĉ ĉ +//// //// +pon ent +Ġ z +Ġk n +le t +D E +re d +Ġf e +Ġ} ,Ċ +Ġ , +( t +Ġf irst +' );Ċ +w ord +Ġ import +Ġa ct +Ġch ar +C T +ĠT r +op le += { +ĉ f +i ent +c ent +. j +le ction +) )Ċ +Ġon ly +Ġpr int +m er +. W +o ck +Ġ -- +T ext +Ġo p +an k +Ġit s +Ġb ack +[ " +Ġne ed +Ġc l +Ġs ub +Ġl a +( ( +. " +O bject +Ġst art +f ile +( self +n er +e y +Ġus er +Ġ ent +ĠC om +it s +ĠC on +ou ble +ow er +it em +ver y +ĠW e +lic k +Ġ Q +ph p +t tp +' : +ic s +Ġu nder +Ġ* Ċ +. L +) ; +ic es +Ġre g +) čĊ +ĉ public +S S +Ġth en +re at +i ous +. G +e k +ire ct +he ck +cri pt +n ing +ĠU n +Ġm ay +ĠW h +B o +I tem +str uct +. st +re am +ib le +lo at +Ġor g +u nd +s um +_ in +.. / +_ M +Ġh ow +r ite +' Ċ +T o +w w +Ġpe ople +ind ex +. n +ht tp +( m +ect or +Ġin d +Ġj av +] ,Ċ +ĠH e +_ st +f ul +o le +) {Ċ +Ġsh ould +op y +el p +i er +_ name +ers on +I ON +ot e +Ġt est +Ġb et +rr or +ul ar +ã Ģ +Ġ Ð +b s +t ing +Ġm ake +T r +Ġa fter +ar get +R O +olum n +r c +_ re +def ine +Ġr ight +r ight +d ay +Ġl ong +[ ] +( p +t d +con d +ĠP ro +Ġre m +ption s +v id +. g +Ġ ext +Ġ __ +' )Ċ +p ace +m p +Ġm in +st ance +a ir +a ction +w h +t ype +ut il +a it +< ? +I C +t ext +Ġp h +Ġf l +. M +cc ess +b r +f ore +ers ion +) ,Ċ +. re +ate g +Ġl oc +in s +- s +tr ib +ĠI nt +Ġa rray +, " +P ro +( c +ess ion +> ĊĊ +Ġs he +" ] +ap h +Ġex p +ert y +ĠS e +Ġp ar +un c +E T +Ġre ad +pr int +Ġre l +Ġfor m +Ġd r +Ex ception +in put +Ġtr ans +#### #### +ord er +B y +Ġa w +it ies +u ff +pl ay +. add +ĠâĢ ĵ +Ġw ant +Ġcom p +ment s +Ġ| | +a z +b e +Ġn umber +Ġre quire +ĠE x +Ġc ol +Ġ key +em ber +Ġt wo +Ġs ize +Ġwh ere +U T +res ult +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ou gh +or ld +o od +u ch +at ive +g er +are nt +Ġ/ * +Ġar g +Ġwh ile +( this +Ġre c +Ġd if +St ate +Ġs pec +r ide +_ F +Ġlo ok +A M +il ity +et er +âĢĻ t +ĊĊ Ċ +ay out +---------------- ---------------- +ag er +Ġc ould +Ġb r +end s +u res +Ġkn ow +et s +ĠI f +ĠS h +. w +b ack +Ġs er +Ġ+ = +Ġf r +() );Ċ +Ġh and +I nd +UL L +I m +() ;ĊĊ +Ġm ost +Ġtr y +Ġn ow +rou gh +> čĊ +ack age +Ġh im +. _ +if y +Ġb reak +Ġ );Ċ +re n +# define +it t +Ġa p +ĉ c +( n +ĠY ou +: ĊĊ +- m +Ġe very +ust om +li ent +oc ument +cri ption +E rror +- b +Ð ¾ +] [ +tr ans +Ġp oint +Ġst d +Ġf il +T ime +Ġm od +Ġ -> +Ġ error +a h +Ġt ext +roll er +lo se +q l +Ġp ol +> < +. B +- c +Ġop en +Ġe st +ĠĠĠĠĠĠĠĠ Ċ +Ġn ext +I M +Ñ Ĥ +O T +à ³ +Ġf ollow +cont ent +ĠĠĠĠĠĠĠĠ ĠĠĠĠ +Ġin clud +H E +ĠR es +Ġh ref +Ð ¸ +Ġc ar +yp es +im age +U n +Ġbo ol +A D +Ġg ame +.F orm +row s +* / +vel op +.D rawing +Ġp ath +is ion +Ġe ach +ĠP l +_t ype +P ath +ne ction +Ġa v +' ). +Ġsup port +EN T +re m +" ). +Ġo wn +Ġc or +c ount +m iss +u ally +Ġm em +st d +i ence +se arch +" ĊĊ +F orm +Ġs ex +en ame +Ġs ign +Ġ et +ĠĠĠĠĠĠĠĠ ĠĠ +', ' +ĠA pp +Ġth ose +o ff +Ġ err +Ġs ystem +Ġbe st +c ode +Ġs ame +Ġd i +us s +Ġc reate +ath er +A rray +. in +f e +S ervice +U N +at s +Ġ Z +al th +Ġm ade +tr ue +A B +Ġm ark +r id +if ied +, čĊ +y n +p ress +Ġg roup +Ġf in +ĠL icense +F ield +eg er +Ġw orld +in ess +t y +Ġpro cess +( b +Ġc re +ar n +iv es +Ġm ain +ide o +_ g +A G +val id +im g +P I +Ġc olor +Ġre port +Ġt ake +ri b +O M +Ġd ay +Re quest +Ġs k +b ers +ĉ s +.A dd +o ot +Im age +Ġcom ple +ol lection +Ġto p +Ġf ree +A S +D e +ĠO n +I G +et a +D ate +Ġa ction +O ver +it or +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +n ot +Ġind ex +h er +ic on +O n +;čĊ čĊ +iv ity +m and +.W indows +O L +Ġre al +Ġm ax +l and +.. .. +r aph +Ġbu ild +le g +ass word +? ĊĊ +âĢ ¦ +o ok +u ck +Ġm essage +t est +iv ers +Ġin put +Ġar t +Ġbet ween +G et +ent er +g round +en e +à ¡ +.l ength +N ode +( i +C lass +f or +ĠâĢ Ķ +t en +o in +Ġ ke +u i +ĠI N +Ġt able +s ub +ĠL e +Ġhe ad +Ġm ust +//////// //////// +. util +Cont ext +Ġor der +Ġm ov +o ver +Ġcont in +Ġs ay +st atic +.T ext +Ġclass Name +pan y +Ġt er +he ad +r g +Ġpro duct +Th is +. âĢĿ +ĠB ut +lo y +Ġd ouble +s g +Ġpl ace +. x +m essage +Ġin formation +pr ivate +Ġo per +c ed +d b +"> +ater ial +ile d +Ġp ut +Q u +Ñ Ģ +un g +m ap +ĉĉĉĉ ĉĉĉĉ +Ġle vel +Com ponent +bo ok +cre en +_ RE +Ġcon fig +ã ģ +O r +. data +Ġd ocument +", " +trib ute +u x +L og +fer ence +p ost +_ e +Ġloc al +and om +ass ert +V al +lect ed +in a +atab ase +A dd +Ġcont ent +.p rint +s igned +r ic +." ĊĊ +Ġf a +! ĊĊ +- f +iv ed +Ġ quest +. ex +Ġf loat +Ġde velop +о Ð +M ap +ad ing +Ġpos s +U E +n amespace +_ O +ĉ b +.G et +> ( +j son +etail s +Ġto o +Ġext ends +ĠN one +Ġf ore +( String +form at +Ġg reat +int er +ca le +Ñ ģ +r on +iv ing +E nt +enc y +x t +o y +Ġmon th +Ġh app +Ġsup er +b ar +def ault +_ de +ord s +l n +( {Ċ +ĠI nd +as es +Ġt itle +Ġcont ext +o h +- p +E m +Ġm et +T est +Ġl ife +_ v +ĠU S +U I +oc ation +m d +Ġ[ Ċ +Ġ ] +s w +Ġin cre +s cript +ent ial +w ays +. de +Ġs rc +Ġc atch +ĠA meric +// Ċ +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ġp ay +pl it +âĢ Ķ +Ġc oun +ob j +.ph p +Ġch ange +eth ing +' re +ast er +lo s +l ation +ĠĠ Ċ +L e +à ¤ +( { +read y +ĠN o +Ġpos ition +Ġo ld +Ġbo ok +able d +b ug +H and +} ;ĊĊ +is play +av ing +Ġgo ver +Ġv ersion +S ystem +n ect +res ponse +St yle +U p +ang u +Ġth ree +in it +er o +Ġl aw +end if +Ġb ase +em ail +( l +_ V +Ġcon f +AT E +Ġd uring +t es +Ġcon sole +ĠP r +Ġs pe +v es +p ath +ial og +d ition +_t o +ard s +Ġagain st +et work +ĠP h +_ L +c ur +im it +W ith +Ġp ower +i um +' ;ĊĊ +Ġw om +le ft +our ces +at ri +ĠI m +ĠM an +or th +$ { +qu als +es e +_s ize +Ġis s +ot al +- g +i que +r ame +Ġw idth +er g +) ( +itt le +T R +ĠThe y +enc es +r l +on s +Ġl abel +. y +- t +up date +an el +s c +.t o +Ġpro ject +à ¼ +Ġe lement +Ġsu ccess +ĉĉ Ċ +.s h +r am +ch ed +() )Ċ +Ġ( Ċ +Ġd ate +Ġto t +_ ST +A ll +ific ation +ĉ var +Ġt ri +ch em +m y +Ġb ig +ĠA d +ĠA t +ot s +n um +A ct +Ġm ap +er a +co pe +. $ +, âĢĿ +Ġp op +Ġf ew +Ġl en +u id +et ers +u les +Ã Ń +s ource +http s +Ġd em +Ġe ar +######## ######## +Ġm atch +or ies +ac es +ĠC l +Ġn ode +ir c +loc al +un ity +} ;Ċ +Ġan other +< < +og le +Ġs it +ew ork +T E +. I +N S +olog y +ou ght +.C ont +> > +Ġc are +st ate +ĉ private +Ġe ffect +++ ) +_f ile +end ing +L ine +F or +i or +ĠS c +Ġf un +.S ize +ĉ else +] ) +st art +v ious +Ġ} , +our s +Ġle g +Ġs ervice +Ġs ince +ir on +L abel +Ġn on +Ġl os +ict ion +Ġf ull +act er +bo ard +g ress +Ġt urn +ith er +.s ize +Ġb ody +res h +et urn +( _ +y les +orm al +p i +Ġsom ething +! -- +u int +Ġpro du +Ġst and +Ġpro ble +Ġav ailable +m t +ĠB l +Ġ ... +Ġb lock +In put +Ġke ep +C ount +op en +Ġ[ ' +Ġth row +uild er +A ction +Ġth ings +Tr ue +Ġ url +ĠB o +print f +Ġre d +j s +.c reate +ĠO r +St atus +In stance +Ġcont rol +Ġcom e +Ġc ustom +loc ation +m odel +Ġ čĊ +Ġs ource +Ġe as +. out +] ĊĊ +one y +Ġaw ait +Ġpart ic +A P +ub lish +od es +_p ro +p ly +rit er +Ġpro v +Ġm ill +H T +] )Ċ +Ġch ang +Ġas k +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +Ġout put +Ġem ail +.p ush +Ġ} čĊčĊ +in ation +atri x +T able +u ccess +] );Ċ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġdis c +( [ +Ġb usiness +he ight +. html +t a +f ield +Ġrequire d +_ R +Ġgover n +} čĊčĊ +le x +. , +ĠS et +ur ch +// / +t s +a f +Ġm ight +ist ory +S tr +Ġne ver +Res ponse +ar se +ad a +ĠH ow +Ġ* ) +Ġ ; +Ġh ard +A d +Ġinter n +us ed +( data +m od +ann el +Ġn p +ug g +Ġ/ >Ċ +Ġcal led +b ody +Ġch o +( r +_s et +ir d +Ġ> = +Ġ} ;Ċ +Ġo ptions +ĠG ener +Ġhe ight +P oint +Y ou +et y +C lick +Ġsm all +Ġ ide +Ġacc ess +angu age +Ġprot ected +Ġj ob +ĠTh ere +D ef +Ġadd ress +Ġu int +N ot +o o +ap s +< div +ain ed +at ur +Ġs um +- w +ĠD ate +Ġl ittle +Ġf ri +Y PE +Ġp ort +e h +pr ing +_p ath +Ġst atus +a im +bo ol +Ġap pe +Ġo s +. name +ens ion +_ G +Ġup date +Con fig +a ff +ER R +Ġ< = +at ely +# if +u ction +ĠT e +Ġl ink +ĠU ser +.f ind +. org +m e +Ġg iven +O ut +# endif +Ġbet ter +P age +Ġfe el +en n +M L +Ġal ready +Ġinclud ing +o ogle +r u +ic ally +pro p +le an +out er +Ġal ways +ord ing +I f +or age +Ġp arent +v is +ĉĉĉĉ ĉĉĉ +Ġg ot +st and +Ġle ss +/ s +ĠA ss +ap t +ire d +ĠA dd +Ġacc ount +p loy +Ġd er +res ent +Ġl ot +Ġval id +ĉ d +Ġb it +pon ents +Ġfollow ing +_ ex +S ON +Ġs ure +oc ial +Ġp rom +ert ies +he ader +.p ro +Ġbo olean +Ġse arch +k en +Ġor ig +Ġ er +E d +E M +a ut +l ing +al ity +By Id +b ed +ĉc ase +eth er +pos it +Ġinv est +ĠO R +Ġs ays +miss ion +AM E +Ġtem p +o ad +Ġre st +in fo +Ġinter est +A rg +Ġper form +pon s +ĠV iew +Ġv er +l ib +( const +U til +List ener +ar ge +Ġm ult +Ġd ie +Ġs ite +../ ../ +E L +Ġval ues +Ġ} )Ċ +p en +N o +ic ro +Ġbe h +Ġ' ./ +ac y +re c +() -> +ĉ ĠĠĠ +" )) +Cont ent +_ W +ple ment +Ġw on +Ġv ideo +ad i +p oint +% % +Ġg l +erv ed +v iron +I F +ut ed +ã ĥ +' m +Ġc ert +Ġpro f +Ġc ell +ar i +Ġpl ayer +a is +Ġc ost +Ġh um +( R +Ġoff ic +k s +.t ext +at ures +Ġtot al +Ġ*/ ĊĊ +o pe +Ġst at +U M +Ġlo ad +ight s +Ġc lear +u ro +Ġte chn +up port +I R +Ġ row +Ġse em +Ġ q +Ġsh ort +ĠN ot +ip p +G roup +se ction +m ax +ir l +Ġover ride +Ġcom pany +Ġd one +" );čĊ +Ġg re +. Re +Ġbel ie +r ist +Ġhe alth +AN T +() ĊĊ +ĠB e +. value +ĠG r +ott om +Ġarg s +P T +st atus +f unc +um ents +- h +N umber +: čĊ +ĠL og +er ver +Ġ) ,Ċ +am ent +Ġob j +in c +Ġchild ren +ic y +I Z +and s +ab ly +Ġdist rib +Ġc ur +er ial +Ġd ays +re ated +re ct +- l +ir m +idd en +om b +Ġin itial +.j s +Ġ â +Qu ery +Ġon line +im al +. con +a u +U rl +cont rol +ire ction +Ġin stance +OR T +ĠF r +wh ere +Ġjav ax +Ġorg an +ap ter +Ġre ason +o ptions +ĠM ar +( a +Ġwith in +.âĢĿ ĊĊ +O DE +_ DE +ad min +end ed +Ġdes ign +ĠD ata +un e +ĠF ile +ro ot +Ġc ent +Ġa rr +_ add +l en +p age +, ' +_ str +Ġb ro +ab ility +ou th +/ c +p ose +irt ual +ear ch +_ url +arg in +H ttp +Ġs chool +av a +Ġcons ider +.l abel +ĠA rray +we b +o pt +.print ln +ul ation +Ġf unc +P L +Ġ" \ +ĠT ext +act ory +(f unction +n ull +Ġen g +d own +Ġin clude +ĠE n +ĠD r +Ġd b +! ! +s ide +Ġin it +quire d +ĠS he +C olumn +re act +Ġan n +Ġst op +Ġl ater +ĠTh at +ent ion +d f +U G +I LE +Ġc lient +ra ft +ff er +PO ST +el per +Ġlo ve +qu ote +ou d +Ġj son +Ġab le +Ġm en +A X +ĠC opyright +à ¶ +av ig +re q +C lient +} );Ċ +.C om +er c +il t +pec ial +_c om +ro om +. Name +Ġg ive +am b +i ke +Ġcon dition +cl ient +ator s +: " +Ġc opy +ut ure +ivers ity +ern al +{ { +ĠC an +ou nc +d o +Ġo cc +Ġapp ro +th ers +z e +Ġe ither +ĠF l +Ġimport ant +Ġle ad +at tr +AR T +E qual +Ġd a +et ch +ent ity +Ġfam ily +add ing +Ġo ption +Ġex ist +ic a +ĠO bject +' ve +v ers +ition al +out put +ĠTr ue +ĠO F +_t ime +Ġof fer +Ġ} );ĊĊ +H ER +eg in +" " +Ġw ater +Ġc he +ĠM y +ore d +Ġst ep +anc es +C K +A Y +à ¸ +str uction +( C +ou ch +St ream +act ive +am a +Ent ity +pro duct +() {Ċ +Ġgovern ment +ĠI D +aj or +A nd +Ġdis play +Ð » +Ġt imes +Ġf our +Ġf ar +Ġpres ent +ĠN S +Ġ\ Ċ +ue st +Ġb as +e cho +ch ild +if ier +Hand ler +Ġl ib +Prop erty +trans lation +Ġro om +Ġon ce +Ġ[ ] +cent er +================ ================ +Ġresult s +Ġcontin ue +Ġt alk +_ get +Ġg row +.s w +e b +ĠP ublic +O P +ec ute +ol s +Ġ ** +" );ĊĊ +Ġm ass +ure d +.c lass +om ic +Ġme an +ip s +Ġa ut +);čĊ čĊ +Ġun til +Ġmark et +Ġare a +u it +Ġl ength +ĠW ith +struct or +e vent +"> < +ĠS p +I V +Ġm us +if f +Ġk ind +a uthor +ound s +m b +_ key +w idth +posit ory +Ġl ight +u k +R ow +oh n +al f +viron ment +app er +ollection s +Ġs ide +_in fo +Ġex ample +im ary +Ġw r +Ġc amp +cri be +" / +Ġm iss +w ay +Ġb ased +Ġpl an +V is +om ain +un k +Ġaw ay +U P +< T +O S +i od +ĠM on +âĢĻ re +Ġli k +à § +iv ely +. v +im er +iz er +S ub +Ġbut ton +ĠU p +Ġexper ience +C L +Ġre nder +_ value +Ġn ear +UR L +al t +Ġcoun try +ib ility +() ,Ċ +e ad +Ġa uthor +Ġspec ific +b ase +( name +on es +ĠD o +Ġal ong +y ear +Ġexp ress +. ' +en v +Ġbeg in +Ġso ftware +Ġim p +Ġw in +ó n +Ġth ing +Tr ans +ĠT HE +Ġ< ? +Ġwh y +Ġdoes n +i j +g ing +ĉ g +Ġs ingle +off set +ar ning +og raph +le y +_c ount +Ġan al +cre ate +/ m +ĠR eg +un ch += $ +is k +Ġright s +( M +Ġ"" "Ċ +ap er +.m odel +Ġp o +em pty +art ment +Ġa nt +ĠWh en +Ġwom en +ĠE d +Ġse ason +Ġde st +à £ +( h +Ġposs ible +Ġse ver +Ġb tn +Ġdid n +Ġs ent +Ġen c +Ġcomm and +Ġ ],Ċ +_ x +Ġre cent +ol ution +v ector +ĠB y +ĠM ay +ĠA ct +» ¿ +Ġm oney +IN T +bs ite +ĉ p +. čĊ +ï »¿ +s l +atter n +ĠC lass +Ġto ld +ud io +c urrent +Ġe qu +Ġa uto +ĠSt ate +d a +ms g +)) ;ĊĊ +Ġwork ing +Ġqu ery +ĠB r +Ġw indow +a uth +on ly +ĉ t +Ġle ast +ag n +Ġex pl +it ter +ar ing +Ġc olumn +ĠGener al +": " +er al +ri or +Ġrec ord +I B +E X +Ġd at +Ġm aking +u ed +ĠC ar +em p +" . +ĠM ed +Ġc lose +Ġper cent +Ġp ast +( g +: ( +Ġw rite +Ġm ove +Ġp at +Cont rol +.T o +Ġv i +*/ Ċ +in ate +' ll +ag ed +N ull +Ġspec ial +IZ E +Ġc ity +/* Ċ +ĠE ng +ix ed +in ary +p y +Ġe ff +ar io +Ġt ell +av or +Ġse lect +le vel +im um +op er +B uilder +I P +') ,Ċ +es c +Ġf ont +" ;ĊĊ +ĠA m +ish ed +ill s +Int er +O W +Ġcour se +Ġl ate +idd le +Ġam ount +Ġas ync +in o +c ul +Ġ ì +and le +_ user +Ġb en +ĠC al +Ġ$ _ +ĠR ep +Ġen ough +T oken +. user +( j +S c +W idth +n ow +at form +Ġlook ing +Ġh old +M odule +IT Y +v o +is on +.D ata +y c +Ġp ot +ĠTr ump +id ual +id es +r t +Ġprop erty +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +am ework +g o +Ġl ow +Ġpar a +Ġpr ice +ur y +Ġto day +ro y +Ġ' / +Ġpol it +Ġ' ' +ym b +P h +Ġad v +Ġatt ack +ĠS te +RO M +an a +Ġme ans +Ġst ory +id s +ak en +Ġme et +Ġm om +ĠâĢ ĺ +Ġ? > +Ġd en +ob ile +ch ange +ĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +ic i +n a +ĠF orm +Ġs ort +Se lect +p are +Ġth ought +_ con +Ġt ask +oc us +ĠD E +ĠM in +Ġo pt +ĉb reak +um er +K E +th en +Ġd et +ĠT est +port s +Ġre view +(' / +m ove +Ġsw itch +ER T +p atch +ann ot +ã Ĥ +Ġab ove +it ive +Ġquest ion +ĠQ u +ãĢĤ ĊĊ +g le +Ġw ord +Ġprov ide +ĠR eturn +Ġre search +ã o +u str +Ġp ublish +chem a +} } +ĠC ON +- in +all back +Ġco ver +\ \ +c olor +ĠI S +Ġwh ether +im ate +is c +B ar +Ġd iv +B e +our n +Ġh aving +le m +pl ayer +ab s +am era +ne y +Ġex c +get her +pl ied +a o +[ $ +Ġ+ + +i pe +sh ow +/ d +[ : +ag ement +le v +_ ID +r ary +ad es +_ se +a use +Ġem ploy +Ġ*/ čĊ +Ġf re +Ġ' @ +Ġcomple t +Ġl arge +r al +\ x +Ġf ac +< String +Ġcre ated +up er +.st ate +Ġh ost +ener ic +/ b +( ! +wh ile +i as +B UG +Ġ );ĊĊ +Ġro le +Re g +ĠC olor +St art +Ġp orn +t op +Ġwe b +Ġde v +Ġde al +++ )Ċ +Int eger +pos ition +. on +Ġ( " +ä ¸ +Ġproble m +s v +Ġp ress +AB LE +AT ION +ĠSe e +an ch +Ġth ough +le ep +Ġ< !-- +Ġpoint s +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠ +. J +Ġ :: +p tr +D B +++ ;Ċ +.p ng +n ode +so ft +pon d +Ġe ver +-------------------------------- -------------------------------- +M enu +(' # +Ġs ervices +p g +} )Ċ +param s +Ġact ually +Ġ" / +Em pty +M ethod +Ġid ent +un ic +Ġmill ion +Ġa ff +st yle +Ġcon c +i os +ign ment +UL T +P r +" ;čĊ +Ġunder stand +u ary +Ġhapp en +Ġser ver +ĠC o +S C +Ġle s +Ġfile s +G rid +s ql +Ġof ten +Ġin fo +_ tr +s rc +on y +Ġsp ace +um b +Ġpass word +Ġst ore +, ĊĊ +ĠWh at +g ed +ĠF alse +U s +sw er +_ index +Ġform at +m ost +s m +N ew +Ġd etails +Ġpro b +ĠAN D +() čĊ +il ar +Ġ$ { +ry pt +.C ollections +$ this +ĠF ree +_ of +(f alse +d ated +Ġ> > +Ġf ace +CT ION +Ġs ave +Ġt yp +de v +(" # +AG E +cont ainer +ed it +Q L +Ġitem s +Ġs ocial +i en +ĠRe act +) .ĊĊ +Ġm ar +Ġre du +ĠR E +.p ut +Ġm ajor +C ell +n ext +Ġexpect ed +Ġy et +Ġin div +trib utes +at is +am ed +Ġf ood +S ource +( string +Ġ+ Ċ +it es +d r +Ġmem bers +Ġcom b +item s +ĠP er +T H += True +Ġb ar +_ SE +com m +( w +)ĊĊ Ċ +Ġs end +Ġin c +un signed +F A +Ġparam s +app ing +ro s +ug in +f a +Ġcon nection +Ġ} ;ĊĊ +Ġbe come +M ode +Ġe v +Ġdif f +ĠUn ited +He ight +ful ly +im ages +Ġm akes +Ġg lobal +Ġcont act +' :Ċ +Ġab s +а Ð +f loat +Ġex cept +ĠP ol +Ch ild +t yp +Ġcert ain +i ón +O UT +Ġim pro +ile s +Ġ-- >Ċ +ĠP art +val ues +os s +/ ** +il it +ĠE vent +cur ity +st er +Ġchar acter +Ġnew s +Ġ" , +Ġde vice +c el +log in +he et +Def ault +@ " +ĉ Ġ +c lick +( value +ĠA b +Ġpre vious +ERR OR +oc al +Ġm aterial +Ġbel ow +ĠCh rist +Ġmed ia +co ver +ĠU I +Ġf ail +Ġbl ack +Ġcom ponent +ĠAmeric an +Ġadd ed +Ġbu y +st it +Ġc ame +Ġde lete +prop erty +od ing +Ġc ard +rop s +Ġhttp s +Ġro ot +Ġhand le +C C +B ack +em plate +Ġget ting +_b y +m ail +_s h +. assert +ĠD ec +( true +Ġcom put +Ġcl aim +' => +ĠS ub +Ġa ir +op s +n av +em ents +( id +Ġent er +ang ed +E nd +Ġloc ation +Ġn ight +Ġdo ing +ĠR ed +l in +}ĊĊ Ċ +vid er +Ġp ick +Ġw atch +ess ages +Ġhum an +Ġd am +p end +d ir +Ġt ax +Ġg irl +re et +Ġbo x +Ġstr ong +( v +re l +Ġinter face +Ġm sg +f ect +_ at +Ġh ouse +Ġtr ack +' );ĊĊ +j e +ĠJ ohn +ist r +( S +ub e +Ġc e +itt ed +V ER +* ) +p arent +Ġapp lication +an y +.sw ing +Ġp ack +\ u +Ġpr act +Ġse ction +ct x +Ġun signed +.P oint +ĠO ne +Ä ± +ip le +a id +Ñ ĥ +V ector +by te +Ġw ait +Ġà ł +à ¥ +Ġto gether +Ġth rows +F O +' )) +h ost +is ing +. view +Ġter ms +fr amework +- r +Ġapp ly +Ġs ession +O ptions +ugg est +Ġo thers +w itter +Ġf und +In it +__ ( +ens or +G ET +Ġsever al +i i +[ j +I O +Ġtem plate +P osition +Ġe con +ach ine +Ġ il +.s pring +m ain +el t +im ent +Re c +m m +ĠUn iversity +urs or +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠ +G L +ict ure +ith ub +c er +c ast +F rom +a les +Ġsub ject +p assword +n y +Ġes c +.w rite +ï¼ Į +Wh at +. H +Ġh istory +ĠF e +Ġindiv idual +un it +Ġ-- > +Ġd u +I ST +Ġus ers +f s +f alse +un t +T itle +Ġm ot +Ġf uture +ach ed +Ġstart ed +Ġm ode +Ġ' < +_ array +Ġa x +'] ;Ċ +i res +Th ere +ug ht +t ml +pos ed +ic ult +Ġto ok +Ġg ames +Ġ} } +Ġ? >Ċ +Ġproduct s +I s +Ġb ad +ĠD es +.p ath +' ĊĊ +ĠP ost +av el +( : +Ġneed s +Ġkn own +F l +Ġex ec +Ġse en +um e +Ġb order +Ġl ive +tem p +P er +Ġvar iable +i et +ĠD ef +Ġg e +em e +_b ack +f irst +Ġprovid ed +//////////////// //////////////// +Ġfil ename +Ġh ope +ul y +a uto +f ind +_ string +b tn +it ude +At tribute +Ġyou ng +.t xt +Ġwe bsite +ĠP rop +Ġe y +> ();Ċ +ion al +AR R +iction ary +ur ther +. +t x +Ġp ur +u el +ymb ol +u ation +ang er +Ġback ground +ec ess +ef ined +.... .... +Ġdes cription +Ġrep resent +") );Ċ +press ion +row ser +Ġser ies +ward s +($ _ +a ise +Ġh ot +ac ity +ri es +action s +C reate +ad io +amp les +Ġorig inal +ens ive +f ont +st ream + using +.spring framework +ser ver +Ġb ill +AC K +il ename +Ġfr ame +Ġ= Ċ +Ed it +adi us +Ġd raw +ank s +Ġd eter +Ġcom es +_ int +Ġfore ach +ang le +Ġe lect +pect ed +He ader +ist ration +F alse +ĠG ame +Ġfil ter +Act ivity +Ġl arg +in ition +Ġ" < +is ed +Ġrem ove +ĠTr ans +m et +se e +Form at +Com mand +ĠE X +N one +Ġfr ont +A SE +ĠR ec +ound ation +Ġv o += \" +( * +Ch ange +.W rite +g roup +i ents +u y +******************************** ******************************** +Ġd ig +h r +( - +Ġg en +n umber +ve c +uro pe +ent ry +L L +Ġst e +Val id +'] , +_p aram +Ġse lected +Ġacc ording +ĠD is +Ġ util +B uffer +_ error +Ġass oci +_S IZE +Ġw or +Ġprint f +r ag + ł +D D +ĠV al +Ġact iv +E ng +et ime +Ġv irtual +a ign +a ur +ĠP res +ĠEx ception +Ġany thing +ĠO ff +Ġh ours +Ġw ar +Arg s +ag ing +Ġmodel s +ĠT ime +O b +am s +j oy +Ġear ly +. read +Ġc enter +ĠIn itial +Ġl anguage +l ength +x y +Ġs n +Ġin f +P ost +Ġag o +Ġeas y +_c ode +ĠAN Y +_ ch +Ġdown load +( T +av ed +âĢ ĵ +Ġstud ents +Ġf ig +l ight +x x +Ġbu ffer +ĠD ep +ĠM ath +IT H +Ġvar i +Ġd ue +F actory +Ġp or +Ġe p +ot ype +Ġcan not +Ġwh ite +< int +ter n +Ġreg ister +Ġpre d +cl us +_d ate +Ġ/ ** +Ġa uth +Ġ[ ]Ċ +Ġper iod +n own +Ġv ot +Ġs creen +' d +T ypes +Ġt mp +е Ð +ur al +Ġben ef +_ y +Ġn et +ĠSt ates +'] [' +ĠN e +ĠN OT +Ġn eg +Ġcomm on +s cope +Ġc red +g es +_T YPE +Ġs uggest +o om +.ĊĊ Ċ +Ġac cept +Ġr andom +er m +ĠV ector +w ith +T ER +( str +Ġres pons +Ġh it +.S et +gr id +ri a +Ġc lick +und le +C ase +ins ert +Util s +Ġ"" " +Ġim plement +at al +tem pt +tem plate +oc r +return s +Ġplay ers +us ers +ed ef +ĠTh ese +Ġam ong +Ġde b +h a +.get Element +Ġc irc +Ġan swer +Ġw alk +Ġt reat +ĠG e +ĠC reate +Ġa ge +Ġre q +O ST +ang ular +Ñ ı +Ġf ive +Ġdistrib uted +Ġfri end +T P +Ġc lean +ow s +.Control s +d is +Ġw ords +. io +z y +Ġhe ader +ĠC heck +âĢĻ m +j ust +h older +=" čĊ +. annot +Ġcol lection +' . +Ġsim ilar +Ġt aken +(" % +Or der +'] Ċ +-m d +ĠT H +ac ed +Ġis n +/ j +Ġs on +gr aph +ĠInt eger +Ġn ecess +re en +Ġ um +Ġ\ < +Ġmom ent +Ġbr ing +Ġind ic +ys is +Le vel +ver se +urre nc +_t est +Ġent ire +D own +Ġ}ĊĊ Ċ +( result +ĠRe ad +à ¨ +M od +Ġtry ing +") ,Ċ +Ġm ember +ĠC or +OD O +- control +un time +ĠS im +D ialog +pl ot +_ on +Ġph ys +} / +Ġn amespace +ĉ čĊ +ac c +Pl ayer +A RE +Ġf oot +Ġbo ard +p art +Ġs us +w ise +ĠM c +Ġp ush +AT A +Ġp lease +ri ed +we et +b it +id ed +V E +ĠS w +U B +Ġt ypes +ed ia +Ġc los +ace book +Wh en +Ġed it +ig ger +Ġen erg +Cont ainer +Ġph ot +ĠC ount +ĠE urope +.I s +ĠR uss +pe ed +ĠS tr +Ġp y +Ġc ult +Ġdef ined +cc ount +Ġob t +.L ocation +Ġth read +il le +Ġinst ead +str ong +ĠS ec +U RE +Ġide a +. se +em y +select ed +Con nection +ac ing +th read +.n ext +Ġc oll +Ġfil m +ist ic +Ġcomp et +Ġcon n +th ough +Ġcom pan +ock et +Ġte ach += ( +Ġph one +Ġact ive +de lete +tr ies +Ġm o +Ġde ath +} );ĊĊ +oc ol +W idget +Ġart icle +ro du +and id +Ñ ĭ +ĠC r +k a +() : +lo od +ĉĉĉ Ċ +Ġal most +Ġs ell +erv let +ri p +Un it +Ġapp lic +Ġcon nect +Ġfe ature +Ġv ia +' ), +Ġl im +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠG u +Eng ine +Ġen s +Ġen vironment +b lock +HER E +N ULL +g y +t ag +) ). +ex p +Ġcom pl +Ġinst all +Ġcomple te +que ue +atur al +Ġgener al +th on +Ġask ed +o res +( res +Ġres erved +S P +ĠâĢ ¦ +Å Ĥ +Ġsign ific +O ff +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠA g +ĠJ ust +ĠE rror +Ġin fl +ad ata +Ġ icon +ask s +' ' +_ LO +? . +ac count +Ġ( * +' )ĊĊ +r ap +_ var +ĠF OR +Ġpart y +ĠY our +c at +str y +. new +bo ot +ĠN ov +Ġv ector +Ġn ormal +Ġf urther +Re pository +Ġd atabase +att le +Ġmus ic +Ġspe ed +Ġd oc +pro cess +IG HT +.p arse +Ġt aking +Ġvi ol +ce ed +ĠA fter +Ġfor ward +Ġc rit +"/ >Ċ +ro t +Ġfa iled +ef ore +Ġconc ern +o e +b a +Ġs ender +Ġter m +h as +=" # +Ġpot ential +N um +Ġpublish ed +.c lose +ĠIm age +str aint +U D +ĠO b +Ġprob ably +l im +" :Ċ +olum e +Ġcon sum +ag ue +ens ions +Ġinvest ig +- year +') ; +-s m +Ġen joy +or ig +er ing +c p +le ased +ple ments +Ġreturn s +p at +B O +ĠH ouse +.L abel +Ġwe ight +igh b +Ġcondition s +Ġex ception +d escription +Ġtr ad +- to +Ġ{ } +Ġmod ule +EN D +. ap +.p rops +Ġcon structor +av es +Ġf avor +ĠN ow +; i +ĠM ain +_ k +er ies +âĢĻ ll +trans form +imest amp +P re +Ġm er +. res +st ant +L ocation +_N AME +Ġlos s +Ġ ĊĊ +n et +Ġeng ine +B lock +Ġiss ues +Ġpar se +ĠB ar +Ġst ay +ĠJ SON +Ġd om +air s +w ner +Ġl ower +", čĊ +ĠD em +uf act +Ġp s +Ġper fect +R L +Ġed uc +l s +em ory +ARR ANT +u ge +Ġex act +. key +al led +e ch +ie f +\ / +o ke +Ġfor mer +al loc +Ġs ix +id a +Ġm argin +Ġhe art +al d +p ack +.getElement ById +ĠW ARRANT +Ġr ather +Ġbuild ing +er man +lic e +Ġquest ions +iz es +le ge +irect ory +Ġj e +Ġc as +pro ps +ut f +Ġse curity +Ġhow ever +we ight +Ġins ide +Ġpres ident +Ch ar +ĠW ITH +.m ap +Ġgr aph +Ġt ag +_st atus +Ġat tempt +op p +us es +ĉ const +Ġr ound +, $ +Ġfri ends +Em ail +? > +Res ource +KE Y +os p +. query +ĠN orth +able s +ist rib +_c lass +el lo +Th at +Ð º +pecial ly +ĠPres ident +Ġcamp aign +Ġal t +are a +Ġch all +Ġop port +.C on +Ġenerg y +li ke +. string +ing ton +) * +y y +Ġprof ession +ir th +Ġse g +æ ľ +Ġh or +i ers +c an +Ġbeh ind +Pro duct +f g +ĠS k +.j pg +? : +] ;ĊĊ +Ġcall back +ĠH ttp +Ñ Į +l ong +M S +AT H +Ġr aise +Ġwant ed +row n +ut or +l t +] = +el ine +M A +Ġse par +c s +se mb +D is +bs erv +ĠW ill +Ġpol icy +Ġth ird +ph one +Ġb ed +/ g +. __ +ĠIn c +iz ing +.re move +in stance +.t ype +Ġs erv +E ach +Ġh ar +ĠM essage +( key +SE LECT +P os +)) ;čĊ +Ġre comm +Ġtr aining +ĠE nt +ĠCh ar +ic ht +(f ile +Ġp rior +G ame +Ġex it +Param s +.c ore +P C +n es +anc ed +( request +P assword +} >Ċ +Ġm ag +Ġre lease +Ġsh all +ud ent +ĠS outh +and o +: ' +.Tab Index +s k +ann er +is set +Ġout side +led ge +Ġ å +ĠR ob +Ġim m +! Ċ +ĠWe b +D es +B C +anc ial +R oute +D ec +fer ences +Ġp urch +ĠM odel +ct or +g n +_st art +_ un +. * +is es +Ġg round +Ġun ique +Ġbe aut +{ " +Ġp our +ĠO ct +Ġt ree +set s +_ res +') -> +_re g +(" \ +Ġby te +B l +Ġd ating +Ġm atter +ĠR em +Ġ' ../ +ĠA ug +ĠL a +Ġ$ ( +ourn al +i am +Ġshow s +w rite +Ġb all +Ġsim ply +Ġf ast +Ġmem ory +A SS +ĠO f +ov ed +ant e +a ul +ist ry +)) );Ċ +Ġf it +< string +Ġpolit ical +anc el +_ . +c ard +.c urrent +o ch +_ image +\ t +# Ċ +( L +Ġindu stry +com ing +Ġex tra +Ġreport ed +.st art +Ġres ources +Ġim g +fl ow +_E X +(n ull +ĠP re +Ġwr ong +inter face +Param eter +n ers +á » +t ure +ers ist +oun try +Ġseem s +al ance +de st +ĉ String +Ġm aint +Ġun it +act ers +ĠT R +if ul +export s +pro ject +App lication +leg ate +Ġt akes +ter m +Ġet c +ust er +Ġappe ar +add ress +Ġf em +h s +Ġh om +, - +Ġdiff icult +Ġcom ing +O pen +Ġset tings +ĠW ar +ĠTh en +Ġaut om +ĠF oundation +Ġqu ite +D escription +Ġb log +i qu +P S +_f ield +J son +SS ION +ĠS ch +ĠL O +Ġdes cri +Ġevery one +Ġpret ty +Ġlong er +Ġm enu +Ġcurrent ly +se c +Ġrelations hip +################ ################ +ĠM ap +as et +Ġparam eters +Ġcr ush +" čĊ +IL ITY +ig ration +Ġc out +t otal +Ġn ames +nd ef +") ; +ri end +yn amic +Ġeff ort +Ġact ual +Ġfield s +O UN +t ers +Ġf ix +_m odel +Ġc ases +C A +M y +Inter face +ĠS E +] ] +al le +ĠN ational +ĠArray List +in line +. V +ar a +ref ix +as c +Re ader +ĠÐ ¿ +ast ic +( () +C l +.annot ation +Ġperform ance +ail y +.to String +.n et +view s +. end +ay ers +l ate +ĠA pr +ed eral +'] ) +.b ody +Ġhigh er +_f l +c r +al ert +_n ode +ĠG oogle +Ġit self +A uth +urrenc y +Ġsignific ant +app end +Ġres pect +str ap +Ġun a +riter ia +P ORT +.ap ache +Out put +Ġpro gress +Ġm id +ĠM icrosoft +Ġres ource +ab lish +Ġd im +. load +.A pp +Ġd irection +Ġadd itional +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ +Ġnum bers +Ġcompan ies +.T h +Ġs ound +user name +Ġstat ement +Ġal ert +Ġcon tract +h ome +_l ength +.Com ponent +e v +. Ex +ï¼ ļ +" ; +ĠH igh +Ġ )ĊĊ +ĠP oint +op h +Ġl ines +-> _ +" )ĊĊ +o x +app lication +Ġ ]Ċ +ĊĊĊĊ ĊĊ +Ġso on +ction s +ing er +Ġj oin +ĠP e +Ġ ë +Ġl as +. E +c ss +/ or +ĠSt art +ĠT O +Ġsub s +con n +com ponents +DE BUG +qu are +F unction +end ar +. index +Ġf ill +Ä Ļ +Ġcho ose +h ow +ĠAmeric a +ass ets +-------- ---- +ĠV alue +Ġoff ice +Ġv eh +Ġtrans form +ĠAr t +Ġin de +Ġf n +Ġim plements +ang o +ple te ++ " +t mp +am ily +Ġhas h +miss ions +E ST +g t +Pro vider +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ġfl ag +Ġpartic ip +d en +ĠReturn s +Ġnot e +ü r +p m +ide os +Ġspec ified +ĠE N +est er +ol id +Ġup on +( std +ĉ v +Ġ' \ +u z +Ġv ert +Ġv ict +ĉ self +Ġ" $ +. k +Ġgroup s +g ithub +l ang +Ġm ut +T O +Ġv e +ĠP lease +;ĊĊ Ċ +ac cess +Ġ{ " +re a +Ġr isk +ick er +og gle +ĉ while +AN G +.s end +Ġwom an +Ġget s +Ġ ign +ĠI d +_ log +ON E +Ġe vid +ĠH ar +_s ub +Ġend l +Ġinclud ed +() );ĊĊ +ĠA p +ig r +Ġs em +ĠBl ack +d oc +_t able +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +- up +Ġca use +Ġ .. +Ġv an +_d ict +Ġf ocus +IN D +CE SS +.L og +Ġmult iple +id o +Ġreg ard +- M +and ler +our se +Ġde g +. U +Ġadd ition +Ġvar ious +Ġrece ive +е н +ĠH T +Ob j +D F +Ġincre ase +ĠO pen +] ; +Ġcomm it +? Ċ +ateg ories +at ory +sh ip +ĠM ich +Ġh tml +rom ise +Ġle ave +Ġstr ateg +av en +ĠCon sole +k nown +- n +_ LE +.com ponent +Ġb re +S ession +i ance +Ġal ign +typ edef +_ result +ĠW HERE +.s plit +Ġread ing +FA ULT +Ġc lo +Ġnot ice +_p r +ar ter +Ġlo ck +Ġstand ard +et ic +ell ow +Ġp adding +ĠH is +Ġst ates +_c ast +( P +a a +Ġintern al +e an +ĠP RO +ĠK ey +Ġes pecially +m ing +Ġc ross +Ġn ational +_ object +f ilter +Ġs cript +. update +_ i +ĠAss ert +/ core +%% %% +Ġproble ms +ist or +Ġ. = +Ġar ch +Ġwrit ten +Ġm ilit +M ENT +. ch +ca pe +ĠM us +_ config +ĠA PI +fo ot +Ġim ages +end l +. In +F irst +Ġpl atform +.pro t +O ption +st e +ĠT ODO +Ġfor ce +. cont +ĉ echo +ĠD av +P tr +( B +R T +ĠB ase +] [' +Ġann ounc +con sole +ĠP y +d s +. as +Ġpre vent +ap an +Ġ{ ' +} ' +Ġde ad +V AL +Q UE +**************************************************************** ******** +Ġch arg +R eturn +Ġf ul +d om +Ġr ules +Ġmod ify +Ġe val +h am +at ement +\ < +ul a += False +R A +Ġcont ains +Ġst ack +m ar +Ġ{ }Ċ +Ġund efined +A ss +ĠCh ina +ve y +* Ċ +Ġplay ing +) / +act or +Ġb ottom +li er +ĠN umber +Ġcou ple +D C +ĠS O +g or +.set Text +s uccess +com mand +F ilter +ĠO ur +_ item +Ġc tx +Ġro ad +V ersion +c ase +ur t +av ior +y ch +semb ly +ĠPro duct +Ġh eld +a fe +Ġinclud es +< quote +Ġa void +ĠF in +ĠM od +Ġt ab +an o +à ± +ipp ing +- e +Ġins ert +t arget +ch an +.M odel +IM E +\ Ċ +Ġm achine +av y +ĠN O +ĠInt er +Ġoper ation +mod al +T ag +] : +Ġprodu ction +Ġare as +Ġre n +_f rom +n bsp +Ġoper ator +m en +app ed +_p er +z en +(" . +.s ave +=" {{ +Ġt or +( response +Ġc andid +Ġcon v +a iled +ĠL ib +com p +ur a +ï¿ ½ +ĠH ere +Ġarg ument +h ood +Ġest ablish +ograph y +Ġon Click +amb da +Ġs ch +Ġmov ie +Ġse c +Ġact ivity +Ø § +Ġs ql +_ all +inc ip +Ġprovid es +Ġs ys +ack et +Ġwas n +Ġus es +ĠF unction +.g oogle +ĠRes ult +Vis ible +ag ma +el come +ĠS y +ĠC ent +AL SE +ac ión +EX T +Ġl icense +ĠL ong +Ġacc om +Ġab ility +. height +Act ive +olog ical +ol y +)) , +.S e +Ġparam eter +pr ite +AB ILITY +.s ervice +ĠG roup +_ query +ĠI tem +in ing +Ġj ud +im s +f ix +ind er +ag ram +Ġfunction s +Ġexper i +ĠE m +Ġro t +Ġp en +.b tn +ĠA S +#if def +Ġcho ice +ĠP age +_P RO +Q U +å ı +ant ity +Â Ń +word s +Ġread only +Ġf lex +prot ected +ĠAn y +Ġchar acters +enc ed +ĠJ uly +il er +C ard +ur ance +Ġre v +.e vent +al y +Ġwon der +ĠP ort +Ġleg al +ro le +Ġt en +Ġgo es +M P +wh ite +): čĊ +)) čĊ +Ġre ference +Ġm is +ĠPro ject +ick s +> & +C ON +Ġre pl +Ġreg ular +St orage +ram ework +Ġgo al +Ġt ouch +.w idget +Ġbu ilt +d es +P art +( re +Ġw orth +h ib +g ame +ĠÐ ² +ac ion +ĠWh ite +(t ype +( ` +Ġn atural +Ġin j +Ġcal cul +ĠApr il +. List +Ġassoci ated +ĉ System +~ ~ += [ +Ġst orage +Ġby tes +Ġtr avel +Ġs ou +Ġpass ed +! = +as cript +. open +Ġgr id +Ġb us +Ġrec ogn +A b +Ġh on +ĠC enter +Ġpre c +b uild +HT ML +ĠS an +Ġcoun tries +a led +t oken +k t +Ġqu al +L ast +ad ow +Ġman ufact +id ad +j ango +N ext +x f +. a +Ġporn o +ĠP M +er ve +it ing +_ th +c i += None +g s +Ġlog in +at ives +'] );Ċ +Ä ħ +Ġ ill +I A +child ren +D O +Ġlevel s +Ġ{ { +Ġlook s +Ġ" # +To String +Ġnecess ary +ĠĠĠ Ċ +c ell +En try +Ġ' # +Ġext rem +Select or +Ġplace holder +L oad +Ġre leased +O RE +En umer +ĠT V +SE T +in q +P ress +ĠDep artment +Ġprop erties +Ġres pond +S earch +a el +Ġre qu +ĠB ook +/ Ċ +( st +Ġfin ancial +ick et +_in put +Ġth reat +( in +Str ip +ì Ŀ +ç ão +Ġevid ence +)) ; +ĠB ro +Ġ[ ];Ċ +Ġ ou +b uf +S cript +d at +Ġr ule +# import +=" / +S erial +Ġstart ing +[ index +a e +Ġcon trib +s ession +_ new +ut able +o ber +Ġ" ./ +Ġlog ger +Ġrecent ly +Ġreturn ed +č čĊ +)) )Ċ +ition s +Ġse ek +Ġcomm unic +Ġ" . +Ġuser name +E CT +D S +Ġother wise +ĠG erman +. aw +Ad apter +ix el +Ġsystem s +Ġd rop +Ġstruct ure +Ġ$ ("# +enc ies +ann ing +ĠL ink +ĠRes ponse +Ġst ri +Å ¼ +ĠD B +æ Ĺ +and roid +sub mit +ot ion +( @ +.t est +ĊĊĊĊ ĊĊĊĊ +] ;čĊ +Ġdirect ly +Ġ" % +r is +el ta +A IL +) {čĊ +m ine +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +( k +b on +as ic +p ite +__ _ +M ax +Ġerror s +ĠWh ile +Ġarg uments +Ġens ure +R ight +-b ased +We b +Ġ- = +Ġint rodu +ĠIn st +ĠW ash +ord in +j oin +D atabase +Ġgr ad +Ġus ually +IT E +Prop s +? >Ċ +ĠG o +@ Override +RE F +Ġ ip +ĠA ustral +Ġ ist +View ById +Ġser ious +Ġcustom er +.prot otype +od o +c or +Ġdo or +ĠWITH OUT +Ġpl ant +Ġbeg an +Ġdist ance +() ). +Ġch ance +Ġor d +c ame +pr agma +Ġprot ect +rag ment +ĠN ode +en ing +Ñ ĩ +Ġr oute +ĠS chool +h i +Ġne ighb +A fter +lic it +Ġcon tr +Ġpr imary +A A +.Write Line +util s +Ġb i +R ed +.L inq +. object +Ġlead ers +un ities +Ġg un +on th +ĠDe v +F ILE +Ġcom ments +_l en +ar row +am ount +R ange +s ert +Grid View +Ġup dated +ĠM o +Ġin form +oci ety +al a +A ccess +Ġh ab +Ġc reat +_ arg +ĠJan uary +ĠD ay +") čĊ +up le +d ocument +gor ith +m enu +ĠO ver +b b +.t itle +_ out +Ġle d +ur i +Ġ? >Ċ +r un +Ġsc ene +( array +de vice +_t itle +ag on +] čĊ +ab y +Ġbe came +bo olean +Ġp ark +ĠC ode +up load +rid ay +ĠSept ember +F e +Ġs en +c ing +F L +C ol +ut s +_p age +in n +Ġim plied +al ing +Ġyour self +.C ount +con f +Ġa ud +_in it +. ) +Ġw rote +N G +. Error +ä » +.f or +Ġe qual +ĠRe quest +Ġser ial +Ġallow s +X X +Ġm iddle +ch or +à ¸ +erv al +.C olumn +read ing +Ġesc ort +ĠAug ust +Ġquick ly +Ġwe ap +ĠC G +rop ri +h o +Ġc op +( struct +ĠB ig +Ġv s +Ġfre qu +. Value +Ġaction s +Ġpro per +Ġin n +Ġobject s +Ġm atrix +av ascript +Ġon es +.g roup +Ġgre en +Ġp aint +ool s +y cl +enc ode +ol t +com ment +. api +D ir +Ġun e +iz ont +.p osition +Ġdes igned +_ val +av i +ir ing +t ab +Ġl ayer +Ġview s +Ġre ve +ra el +ĠO N +r ics +n p +Ġc ore +() );čĊ +M ain +Ġexp ert +ĉĉ čĊ +_ en +Ġ/ > +ut ter +I AL +ail s +ĠK ing +*/ ĊĊ +ĠM et +_ end +add r +or a +Ġ ir +M in +Ġsur pr +Ġre pe +Ġdirect ory +P UT +- S +Ġe lection +h aps +.p re +c m +Val ues +Ġ" Ċ +c olumn +iv il +Log in +in ue +Ġbeaut iful +Ġse cret +(e vent +Ġch at +um s +Ġorig in +Ġeffect s +Ġman agement +ill a +t k +Ġset ting +ĠC our +Ġmass age +ĉ end +Ġhapp y +Ġfin ish +Ġc amera +ĠV er +ĠDem ocr +ĠH er +( Q +con s +it a +Ġ' . +{ } +ĉ C +Ġst uff +Ġ :Ċ +ĠA R +T ask +h idden +er os +IG N +at io +ĠHe alth +ol ute +Ent er +' > +ĠT witter +ĠCount y +s cribe +Ġ= >Ċ +Ġh y +f it +Ġmilit ary +Ġsa le +re quired +n on +boot strap +h old +r im +- old +ĠD own +Ġm ention +cont act +_g roup +od ay +Ġto wn +Ġsol ution +u ate +ell ing +] -> +ot es +ent al +om en +osp ital +ĠS up +_ EN +Ġsl ow +SE SSION +Ġbl ue +ag o +Ġl ives +Ġ ^ +. un +in st +en ge +Ġcustom ers +Ġc ast +ud get +ï¼ ģ +ic ens +Ġdeter min +Se lected +_ pl +ue ue +Ġd ark +// ĊĊ +s i +ther n +ĠJ apan +/ w +P U +ĠE ast +ov ie +Ġp ackage +Ġn or +Ġap i +b ot +" ];Ċ +_p ost +ul ate +Ġcl ub +') );Ċ +Ġlo op +PI O +ion e +sh ot +In itial +Ġplay ed +reg ister +rou ght +_m ax +ac ement +m atch +raph ics +A ST +Ġexist ing +Ġcomple x +D A +.C h +.com mon +m o +Ġ' ../../ +it o +Ġanal ysis +Ġdel iver +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +id x +à ł +ong o +ĠEng lish +< !-- +Ġcomput er +EN SE +Ġp as +Ġr ais +H ash +Ġm obile +Ġo wner +F IG +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +th es +Ġat tr +w d +.t ime +aw n +Ġtreat ment +ĠA c +. View +im pl +m ore +p ass +Ġh a +.f rom +Ġle ading +FF FF +( error +. ui +at ar +ad ers +d ates +Ġz u +Ġfl ow +T arget +Ġinvol ved +Ġi o +par se +$ _ +he st +. int +- item +as y +S p +Ġsh ift +N T +Ġt f +_T R +. web +C S +Ġ} ) +Ġey es +_ z +' );čĊ +if orn +Ġ{ @ +Ġn ice +.l ist +ĠĠĠĠ čĊ +Ġf loor +Ġred irect +ĠU K +( [' +Ġw ish +Ġcap t +leg al +ĠI O +Ġst age +. String +ĠA fr +ig en +ĠS H +De lete +ell s +Ġsol id +Ġmeet ing +Ġwork ed +Ġed itor +in y +Ð ¼ +_ read +. Id +e ff +Off set +ch a +US ER +ĉĉ ĠĠĠ +ipp ed +Ġd ict +ĠR un +.h pp +Ġan g +x ml +im ple +Ġmed ical +_t oken +con nect +Ġh our +Ġcont roller +_m essage +U ID +G r +and ed +_C H +Ġbook s +Ġspe ak +am ing +Ġm ount +Rec ord +ĉ struct +.W eb +ond on +Ġ// Ċ +Ġf elt +.A uto +id ge +_p os +P R +Ġmod ern +C ollection +_m sg +C D +ĠL o +Ġsecond s +ib ly +.e quals +Ġintern ational +# pragma +oo th +W riter +i ate +Ġce le +ĠB it +iv o +iv ery +r d +HE CK +Ġc ache +.c ount +Ġro ll +.Re ad +RE D +Ġset up +izont al +model s +arg v +Ġconsider ed +=" ../ +set tings +ĠR el +Ġgrow th +Ġm ix +ĠWash ington +Ġpl t +ĠI M +á º +Ġturn ed +ĠDate Time +ĠW ed +( url +Ġ" - +Ġlet ter +As ync +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +ĠOct ober +_l ine +Ġatt ention +Ġcol lect +ĠH ash +Ġim ag +T ree +Ġsit uation +et te +_n o +IV E +Ġv on +.t arget +Ġknow ledge +Ġdr ive +.p ost +Ġb lood +Ġc it +pr imary +Ġconfig uration +te e +Ġph oto +is ode +Tr ace +Ġg ave +Ġsh ot +ĠA ir +Ġm other +pr ice +Ġmor ning +)) {Ċ +- x +Ġtr ade +Ġdes c +Ġ&& Ċ +Ġparent s +A pi +å Ī +t ed +w er +Ġ æ +Ġs y +ĠK e +Par ser +å ħ +anc y +Ġpie ce +iforn ia +to String +r an +id ing +PT ION +com es +/ lic +.c lient +E l +L ong +Ġprofession al +ru pt +v a +Ġcomplet ely +Ġpract ice +Ġse lection +R em +in i +Ġc am +RE E +Ġsit es +p a +AT US +Ñģ ÑĤ +arr ant +* ( +_ KEY +ĠB utton +ĠF riday +se qu +Ġre ader +Ġm essages +è ¯ +Ġbu f +K e +Ġn ov +H P +M sg +al ign +ar ily +Ġ' , +_w ith +Ġd as +Ġhe ard +at omic +ri al +) [ +Ġdis e +@ end +Ġg old +Ġf air +Ġsa les +. Button +str ict +s ave +Ġme asure +Ġ" + +ec ause +View Controller +ĠT able +.p aram +Ġdec ided +(( ( +IN FO +Ġopport unity +T e +IC ENSE +cc ording +k i +ĠU N +Ġcont ain +Ġman ager +Ġp ain +ĠF ire +rom e +Ġpl ans +F ound +l ay +ĠDec ember +Ġinfl u +à º +ren ch +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ġ +az ing +b rief +c all +wo od +Ġload ed +Ġgr and +/ f +im p +_ U +ST R +âĢ ¢ +Ġcred it +.C olor +or ge +QUE ST +Ġdiffer ence +ĠP C +w args +Ġp ub +und ay +Ġf ra +.m ax +Ġtri ed +ann els +s end +Ġreport s +Ġad ult +ä º +Ġcons ist +ĠSt reet +ĠPro gram +S QL +M atrix +ounc il +- A +ĉ w +Ġwho se +Ġrel ig +ĠS ex +Ġg ives +n one +.m essage +( G +.aw t +- right +ĠNov ember +ell ig +ut ive +Ä ĥ +over n +Ġeas ily +Ġide as +ĠÐ ½ +/c ss +ly ing +el le +C an +_c olor +оР² +Ġp air +ng th +Ġs plit +d rop +art y +on a +Ġcap ital +Ġhe ar +Ġex ists +ĉ log +em o +R un +o i +Ġpar ser +ĠM ethod +Ġeduc ation +[ k +Ġlib rary +> ";Ċ +_ UN +ĉ std +od ed +Ġcall s +h ere +R el +Ġbr and +back ground +g a +_add ress +_param s +C ategory +ĠInd ia +_e vent +Ġ ing +R ender +.c l +ump y +Ġp et +F C +ĠA nt +Ex t +Ġchar ge +en ed +gr ad +E O +Ġdep end +Ġ .ĊĊ +fr ame +Ġd f +Ġh uge +ĠP ART +ed s +; ; +ĠA M +Ġbas ic +ĠL et +lic h +Ġar m +Ġst ar +Ġf ederal +W ork +Ġcar ry +ĠIs rael +( obj +={ { +Ġs aved +Ġs yn +Ġconst ant +V ENT +Ġpos itive +Ġcon duct +Ġsk in +Ġear lier +Ġl ayout +ĠI P +O UR +Ġt im +styles heet +_ cl +ĠC ard +++ ){Ċ +Ġtem per +ĠDav id +ĉ try +.d art +Ġwant s +Ġp icture +Ġv ideos +ĠCom m +is ions +_M AX +M apping +- content +ĠE ar +- de +Ġpre m +br uary +Ġcom ponents +Ġthrough out +Ġp ull +Ġp ages +ent e +res pond +Ġg as +cript or +Ġed ge +Ġb ound +A CT +**** ** +Ġcre ating +ĠC H +Ġnull ptr +B r ++ ' +.c o +> :: +Ġle arning +.L ength +_S H +Ġpat ients +A IN +Ġk ids +Ġcom fort +Ġsh own +ug ins +ĠB ack +ell a +_C L +Ġl at +Ġdis patch +Ġclass es +. at +.b egin +Ġsuccess ful +b an +Ġobt ain +ĠS l +Ġl ack +iter ator +Th read +(s ize +Ġn one +.h as +_ X +s ort +n ap +p et +b in +ĠCan ada +The y +Ġd ans +ĠM at +< td +Ġh air +Ġ' ',Ċ +Ġc u +Ġlaw s +let ed +p ed +Ġp ow +Ġk new +_C OM +_ , +ĠM ag +id ents +( req +Ġ ), +- center +Ġw ide +ĠA uthor +st ants +Ġjob s +Ġm ath +et imes +Bo olean +Ġs cope +_ is +Ġme as +Ġkey s +el ay +Ġexact ly +'=> ' +ĠP aul +m as +ĉ print +(l en +f d +Ġ) ; +. Event +q li +ir it +ield s +om an +ĠT op +Ġv ote +Ġm ask +Ġthem e +- Ċ +Ġpro ps +Ġf ine +Ġwrit er +_ offset +c ar +Ġal tern +Ġc opyright +Ġdest roy +pp er +Ġgener ate +pp ed +âĢĻ d +ĠĠĠĠĠĠ Ċ +m ake +ĠSh ow +Ġb rowser +Ġfavor ite +Ġcare er +Ġhappen ed +( char +Ġrecomm end +Ġl iter +.f ilter +gr ade +Ġ £ +Ph one +om s +Ġn amed +- label +ip o +ĠO ther +Ġp anel +Ġro ck +S cale +ĉ assert +Ð ´ +Ġtr ust +fr ont +Ġdem on +A r +N et +Ġecon omic +foot er +Ġr ace +(n ode +ĠO ption +s plit +Ġphys ical +if est +Ġrem oved +. http +)) ,Ċ +Ġlook ed +' ; +d ing +g est +atur day +/lic enses +Pr ice +Ġd ro +Ġto wards +Ġun s +ĠC L +ĉ static +Ġ rows +Ġdef ine +.re place +Ġf ather +ĠDes ign +ass ign +m ut +De vice +D id +') )Ċ +omet ry +ay load +Ġh istor +ĠP aram +ĠBo olean +Ġn ature +Ġj s +Ġn ation +i h +Ġdis cover +se m +Hand le +ĉ r +ĠTe chn +Ġw all +{ $ +@ property +Ġ" ../ +Ġex am +.d raw +opp ing +Ġnear ly +Ġco ol +Ġinde pend +RE S +Ġhand ler +ĠMon day +Ġs un +St yles +ous ly +Ġ ĉ +v est +D isplay +( y +atic ally +Ġpred ict +y ing +Ġsom etimes +" ]Ċ +Ġdr ink +Ġb ul +ific ations +. insert +.re g +Ġtest s +Al ignment +Ġal leg +Ġat tribute +ĠN ote +Ġmy self +art s +N ow +Ġinterest ing +li ents +Ġpop ulation +ĠCal ifornia +" I +å ¹ +Ġgre ater +ues day +Ġth ous +Ġcost s +Ġla unch +\ Http +k er +b and +ĠPl ay +Ġb and +.sh ape +es ome +art icle +.r f +Ġw er +á s +em bers +us r +B A +ic an +et t +valid ate +ult i +Ġimmedi ately +z er +Ġfig ure +o es +ell er +irc le +ĠS ign +.d b +Ġr ank +By tes +Ġproject s +_re c +UL AR +A PI +ĠL ine +P ort +Ġp oll +Ġg iving +id ence +-- Ċ +Ġpl ot +ic ial +Ġw arrant +IT ION +ĠD ouble +Ġbill ion +gorith m +Ġequ ipment +D ATE +Ġ@ " +E E +Ġp le +i ation +Ġhead ers +Ġpro ced +.Component Model +ĠOb ama +Ġp a +ĠB est +im ately +.get String +. \ +mp loy +Ġr aw +_b lock +und red +" },Ċ +.Group Layout +Ġb rought +NS String +th row +cre ated +.N ew +_ view +C P +ep s +O p +Ġgr atis +Ġ' " +Ġinter view +"" "Ċ +Ġpart ial +Ġa ria +b ing +A uthor +Bo ok +ĠP at +um an +Us ers +pl us +ĠD irect +ven ue +al pha +UC CESS +ĠC all +Ġ );čĊ +im ated +Ġrem ain +Ġant i +ĠL ondon +Ġsaf ety +PO SE +o les +cont roller +By te +ĠCour t +ĠPh il +ĠAss oci +en a +å IJ +_ST R +co in +resh old +Ġb atch +_C lick +entic ation +> ';Ċ +ent y +Ġbegin ning +Ġz ero +ĠCon vert +Ġt err +Ġp aid +Ġincre ased +c atch +-s ize +act ivity +e quals +Ġque ue +Ġ" ' +ĠIntern ational +Ġf ür +urs day +Ġsc ient +all ow +ax is +Ġapp ropri +ed ge +Ġid x +S uccess +ent ifier +: \ +x is +Ġmax imum +ark s +Ġb irth +( index +Ġmay be +.p y +file s +Ġlim ited +_ check +lo ok +pl ies +Ġmov ement +'] . +Ġbro ad +ĠB E +ĠUn ityEngine +.c pp +ĠE very +Ad min +Ġf ans +p ared +Ċ ĠĠĠĠĊ +Ġfore ign +Ġp an +Ġt our +ĠOr der +Ġmov ing +Ġa uf +C all +c b +Å Ł +vent ory +ĠS ql +Ġful ly +Click Listener +W ORD +Ġannounc ed +) čĊčĊ +Ġagre ed +ri e +Ġe arn +_l ink +. array +(t ext +Ġmaterial s +, p +ff ff +v g +Ġ © +Ġun less +aj ax +LO G +Ġsex ual +Ġ\ " +- time +Ġco ach +Ġsupport ed +Ġphot os +if orm +.C reate +) ] +ri er +Ġd ialog +av er +ig e +) + +_id x +: [ +_m in +ĠC ong +Ġpress ure +Ġteam s +S ign +b egin +ri an +NE SS +L S +Ġimpro ve +ĠS unday +Ġdef inition +ig er +roll ers +Ġthink ing +T emplate +- F +Ġem erg +pl ates +ĠUS A +.set State +ĠAl so +re v +Ġen able +ĠC O +PE CT +Ġcon cept +) - +ĠâĢ ¢ +Ġset s +Ġmean ing +em on +ĠCon s +c mp +ed er +ann ed +icens ed +ĠS uper +Ġd aily +Ġmult i +_ u +Ġchall eng +_m ode +ĠP romise +Ġstr ict +j o +int on +( list +On ly +> { +Ġveh icle +í ķ +ĠPl ayer +ĠD el +Ġp ool +. url +nes day +();čĊ čĊ +Ġ" );Ċ +L ocal +. ");Ċ +Ġorgan ization +re nder +ĠApp lication +Ġsum mer +ex pected +N A +Ġr ap +_ obj +Ġsur face +ĠP UR +Ġ}, ĊĊ +Ġvariable s +(m essage +Ġop in +.b ack +а н +Ġwork ers +v m +C o +ught er +Ġm aster +Ġ" ", +Ġst ories +. User +Ġcele br +ines e +B S +ĠCom mand +ash board +Ġo g +k g +. image +.st yle +Ġstep s +ĠB en +( args +ĠP erson +, y +Ġofficial s +| Ċ +Ġsk ills +v c +Ġbuild er +Ġg ar +A ccount +ĠA uth +ç Ķ +'] )Ċ +ĠA T +n n +. Int +SS ERT +Ġeffect ive +LE TE +Ġto ols +AR D +Ġdig ital +D ouble +ĠF ind +R C +Ġin line +/ r +AR AM +AS K +Ġint ent +a ight +_add r +Ġrequest s +.f irst +Ġde bug +Ġsp ent +() ));Ċ +Å Ľ +Ġpr incip +Log ger +clud es +. use +Ġsur v +med ia +ĠFe bruary +ĠM ac +Ġmiss ing +Ġw ife +Ġtalk ing +ĠM ake +Ġc art +Ġloc ated +E nc +- a +ch ron +Ġc ards +Ġgu y +Ġp ers +ĠY es +ate ver +ĠA ng +ol ar +ĠE ven +Ġacc ur +ĠP ower +ĠG old +c lear +Pro cess +Ġrec ords +Ġk illed +.c lear +ĠWARRANT IES +Ġpur pose +pan el +J ECT +ÃŃ a +Ġex erc +W S +/ L +. exports +Ġ__ _ +Ġs in +S ervlet +Ġd é +.de lete +ro ke +S l +ug h +ear s +Ġpoint er +Ġh op +all ery +Ġo bs +co very +ĉ char +ĉĉĉĉ ĉĉĉĉĉĉ +ĉ def +oc ity +itch en +ul ations +ĠF IT +Ġ ). +straint s +vent ion +Ġrequ ires +ĠO per +M E +OUN T +al let +Ġn orm +I RE +ex as +Ġprogram s +Ġwe ak +' .$ +u ing +ĉ ĠĠĠĠĠĠĠ +Ġm il +Ġf irm +init ely +_VAL UE +ap se +atis f +Ġdem and +_m od +Ġdescri bed +Ġpl aces +V ID +Ġal one +Ġex port +Ġv ec +ĠM ax +Ġactiv ities +ict ures +g ener +Ġm a +Ĥ ¬ +Ġexpress ion +C allback +_ content +ĠM ost +Ġtest ing +E C +CH ANT +Ġad just +.Th reading +( ctx +Ġag ree +ig hest +Ġu i +ĠL aw +. Y +> ĊĊ +.ex ample +ber g +Ġmov ed +ĉ e +ĠS aturday +Ġpay load +Ä ĩ +) :ĊĊ +Ġbe y +ur er +< script +Ġs ymbol +Ġass um +Ġp ul +E ffect +Ġh undred +To ol +ak ed +con nection +Ġvo ice +Ġp d +Ġtrans action +Ġlink s +E rr +ĠInd ian +T C +atal og +n i +s ign +<< " +j i +y a +Ġdemon str +ul ated +. St +Ġinst it +Ġbo ost +Ġcell s +ol ic +.P ro +: , +"> \ +Ġth us +ĠReg ister +h ol +ĠCh inese +Ġpost ed +Ġm agn +ab ilities +Ġdise ase +Ġrem ains +ĠPro f +- form +Ġc in +org an +ic ate +Ġst ress +] * +Ġ ---------------------------------------------------------------- +_ context +or ry +Ġd ied +m at +Ġstart s +.M essage +Ġrun s +Ġgu ide +Ġwarrant y +ential s +d ict +ĠS ize +ul er +Ġrespons ible +_SE T +Ġcont aining +ĠPr ice +| | +F S +Ġem p +_b utton +( uint +Ġsu ff +p th +Ġdef initely +put e +Ġmarket ing +ĠW H +ĠS ie ++ = +OL OR +Ġcons ult +Ġs igned +Ġse quence +le e +Ġrequire ments +h y +Ex press +M T +se y +Ġ ult +å ® +ellig ence +Ġanal y +Ġd ress +eng ine +ĠG reat +ĠAnd roid +ĠA lex +m ode +D ictionary +.D ate +ä ½ +V ICE +Ġfam ilies +ĠRuss ian +ĠT imes +.c all +$ ( +Pro file +Ġf older +ch es +Ġleg is +_ row +un es +Ù Ħ +Ġ} ). +Ass ert +ag en +ĠH and +I ter +Ġbig gest +ore ach +Ġpol ic +Ġper missions +Ġshow ed +ĠE lement +Ġtop ic +âĢĶ âĢĶ +ro ad +ĠB ank +rec ord +Ġpart ners +ĠR ef +ess ions +Ġass ess +U ST +ĠPart y +pro du +L C +Ġ ul +. form +h ide +c opy +UT F +ĠSO FTWARE +čĊčĊ čĊ +ĠL in +un a +ug ar +Ġadmin istration +Ġopen ing +Ġsc an +Ġcontin ued +com ponent +.s p +Ġhapp ens +um my +ĠP R +.F ile +ĠDown load +Lo ading +d i +Ġwait ing +_A DD +T ab +.query Selector +Ġecon omy +ĠF rench +t xt +Ġf ant +_ ;Ċ +H older +S H +Ġn umpy +Ġst reet +Ġm ale +\ Model +ang ing +ĠB ill +Ġprevious ly +B I +ĠSec ret +Ġm ist +ĠF ield +up s +ĠPro cess +Ġke pt +ĠO T +Ġtrad itional +. i +am in +Ġhelp s +An y +orig in +ilt ers +j u +d esc +ĠA ccount +Ġ) čĊ +k top +ol ly +Ġf s +Ġ ê +Ġ ut +Ġcent ral +(t est +.A n +Ġs atisf +G R +ĠF ull +Ġhe at +ib er +Ġon to +m os +S chema +Ġfact ory +" .$ +aw s +St atement +(t arget +ĉ new +.b e +Ġg uest +Ġm al +AR Y +Ġre ached +Ġm ouse +Ġchall enge +ĉd ouble +ĠT em +Ġt error +Ġex tract +_T O +Ġsepar ate +Ġm ir +h elp +Ġcap acity +ĠProp erty +k an +_c reate +ĠL ight +.p arent +Ġunderstand ing +Ġeas ier +Ġ| = +Ġen h +Ġf at +Ġprot est +am m +_ AT +- of +il s +ĠO h +Ġps ych +Ġ$ . +ind s +Ġrel ative +sh op +sh ort +ĠS and +uest ion +Ġf ear +/ ĊĊ +. context +Ġschool s +Ġser ve +z one +_d b +Ġmajor ity +ex ample +Ġl ang +ĉ ĠĠ +Reg ister +end o +Ġprocess ing +_t emplate +- user +Ġe g +C OM +ĠBl ue +i ro +Ġrem ote +ĠI T +#! / +Ġred istrib +ra z +ĠS ince +ĠT ur +Back ground +== = +Ġref lect +Ġpro s +c md +Ġwh om +Com pat +ĠA re +Id entifier +ĠTh om +_ port +g u +Ġmon itor +r m +Ġpat ient +ver ter +Ġg ain +- ui +In st +Ġd ies +A rea +_f ilter +Ġgr at +Ġreal ity +ord inate +ol ved +Cont act +Ġcompl iance +_ or +ĠV ar +d l +Ġapp end +G ER +(m ax +.re nder +Ġd ynamic +ordin ates +_ options +_c olumn +Ġb atter +s pace +L a +ĠS ource +/b in +Ġd os +ĠBo ard +ĠTh read +ĠA L +( config +ĠM er +Ġm iles +_ header +ETH OD +iz z +Ġbenef it +Ġinteg r +(c urrent +ul o +. default +ĠD iv +Ġt on +o th +erv ation +ed om +Ġb aby +ce ived +.t op +rior ity +ĠL ocal +ri age +Ġattack s +Ġh ospital +Ġfem ale +ĠLog in +ĠFl or +Ġch ain +ash ion +Text ure +S ave +Ġf arm +.cont ains +.T est +Ġknow s +Ġgener ally +ip eline +Ġme ant +enc ia +Ġn icht +Ġcont ents +P M +ched ule +( line +C G +j ob +ĠRe al +u er +f irm +Ġ Ø +et ro +" `Ċ +Ġspe ech +Ġth r +fore ach +Ġw arn +ĉ l +Ġhe avy +< li +N e +Ġinvestig ation +M ath +- title +Ġch urch +Ġdes pite +ch ain +Ġwh atever +ar ian +f n +Ġm eta +} )ĊĊ +U FF +Ġregard ing +_S UCCESS +m es +ĠInt ent +Ġres olve +pos s +ir a +for ce +o ice +à ¢ +Ġp m +Ġup dates +A rr +Ġ Ñ +test ing +Ġto ward +nt ax +ë ĭ +Ġlist en +Ġgo als +Instance State +D r +Ġr are +Ġtr ail +Ke ys +C al +C ar +ĠPe ople +ĉ local +class es +Re ference +.for Each +em b +act iv +Ġpr im +red ict +Ġr ad +æķ ° +.B ack +Ġsp read +Ġc lock +Ġv ir +ed itor +Ġeffort s +Ġbr anch +Ġind ust +Ġmot or +Ġam b +Ġdat etime +Ġren cont +ĠChrist ian +ĠAmeric ans +f ull +Ġf mt +.m ain +Ġca used +_ update +ĠCont ent +AT CH +Ġb ath +ĠE ach +Ġr adio +ach ment +uz z +Sub mit +Ġre strict +ab in +ĠL oad +Ġext ension +Ġess ay +Ġh at +avi our +to Be +": [ +Ġoffer ed +Ġv ill +(d ouble +æĹ ¥ +b c +_f ree +ĠM iss +ĠB er +Ġ è +ĠL ike +Ġhelp ed +.get Name +_ AL +Ġsp irit +ĠAp ache +w s +Ġthere fore +( params +_ img +Ġpe ace +Ġinc or +ĠEX PECT +Ġmin or +ip es +ĉ data +select or +c ity +tr ie +.b ase +_f rame +Ġopen ed +/ json +L Y +n u +.D e +t f +m argin +.P arse +Ġp i +Ġe q +b d +Field s +ĠT ree +Ġb an +ist an +Ċ ĠĠĠĠĠĠĠĠĊ +ĉg l +Ġprodu ced +s ystem +M ark +_h ash +Ġb g +Ġconst it +ĠLe ague +Ġmiss ion +_ format +([ Ċ +clus ion +! " +Ð · +b reak +ĉs witch +Ġth er +Trans form +Ġfoot ball +- link +r oute +. auth +Ġb ag +ov ers +Ġen abled +Ġr ac +( I +C R +anc ing +Ġman aged +_ q +NG TH +Ġm ac +ĠA uto +ament e +Ġ' ', +.App end +Ġp in +. item +ack ing +Ġocc as +p erson +Ġt i +.Re g +Ġh aven +Ġg lass +Ġ" ) +_ char +res ource +Ġep isode +Ġ' _ +ĠE s +ĠEar th +Âł Âł +UP DATE +ĠS ou +u is +t ypes +Ġm as +Ġf av +Ġcon struct +_r ate +er as +Ġ| Ċ +rop erties +Ġext ernal +Ġap plied +Ġpre fix +ot ed +l ers +Ġc old +ĠS P +ĠCh urch +ĠOut put +los ed +ç ļ +ific ate +oper ation +her it +x FF +. env +_ err +os h +D irection +C ancel +ĠFr ank +Ġfind ing +. )ĊĊ +Ġr outer +ãĥ » +s es +Ġc row +== ' +Ġs and +Ġr id +it ure +Ġent re +Ġo bserv +Ġv ac +ð Ł +- T +A rt +n ight +. search +Ġex change +Ġdistr ict +. os +Ġdep artment +Ġdoc uments +Ġcent ury +ĠN ext +H ost +ĠK IND +Ġsus p +- P +re nd +. em +u ite +ist ers +( json +ĠAn n +w t +at i +ĠHT ML +wh en +D irectory +Ġsh ut +< a +ed y +Ġhealth y +Ġtemper ature +ĠG en +Ġmet al +Ġsub mit +ĠD O +Ġat tract +Ġ{ };Ċ +ĠW ord +Ġl l +Ġseem ed +k o +I ED +Ġl abor +.Cont ext +Ġas set +y ou +Ġc ars +ĠC olumn +Ġr é +Ġs quare +ĠNS String +âĢĿ , +ap es +.. .Ċ +Ġthan ks +( props +Ġt ick +Ġexper iment +Ġpr ison +t ree +- text +ĠIO Exception +-w idth +_ST ATUS +f ast +-b ody +- header +Ġgu ar +cre te +ĠT im +Ġclear ly +ĠRepublic an +Ġjust ify +и ÑĤ +ĉ ĠĠĠĠ +c ache +; // +Ġpres ence +Ġfact ors +Ġemploy ee +] )) +M ember +Ġselect or +b or +ĠM ex +çļ Ħ +ut ex +_t ag +ail ure +ĠN et +Ġre li +E G +Ġf printf +Ġte en +lo ss +Ġle aving +De legate +Ġbe at +Ġmin ute +sub scribe +Ġredistrib ute +Con stants +Ġcan cer +/ { +B L +Ġs pan +ĠCh ild +C enter +Ġear th +Y S +ĠLe vel +Ġse a +.s upport +.in ner +. Item +ill ing +ĠĠĠĠĊ ĠĠĠĠĊ +ĠL abel +ĠE st +( arg +bo Box +ĉf oreach +c os +F ailed +sw ers +Ed itor +r ont +ĠM P +ex pr +ĠL ife +Ġ? ? +ö r +Ġatt end +ĠQ ue +Ġspec ies +- D +Ġa us +Str uct +Ġadvant age +ost on +-b lock +in itial +C RE +Ġtr uly +Ġcomp are +or ney +Ġs pect +F ull +b es +Ġvis ible +Ġm ess +st ances +Ġcl oud +_v ersion +Ġf urn +ic ago +LO W +Ġtraff ic +Ġf ol +rypt o +Ġdecl ar +Ġsl ot +ĠEx t +ĠEng land +ĠU nder +Ġt a +let ter +Ġoffic er +ĠDon ald +Y es +_ json +IT ableView +ĠU SE +mploy ee +Ġopin ion +ĠA ut +b order +Ġad vice +Ġautom atically +is co +Ġm m +. vis +am l +Ġinitial ize +Ġ( { +Ġ ;ĊĊ +Ġgener ation +Ġb its +clip se +Ġun f +ut ors +pl t +Ġdel ta +est roy +is is +< br +Ġlimit ations +Ġend ed +ĠM ad +il m +Th ese +ĠMin ister +Ġch art +F ragment +Ġindepend ent +Y ear +Ġin str +Ġt ags +A VE +ĠAr ch +st op +Pro gress +Ġm i +Ġlearn ed +G e +Ġhot el +S M +T YPE +Ġc y +ERS ION +un ately +l imit +s el +Ġmov ies +Ġste el +o z +g b +ĠC amp +s ite +ĠLog ger +P LE +оР´ +. right +ĠC ore +Ġm ixed +st ep +Ġput s +s uper +R outer +. Http +ly ph +ĠColor s +Ġandroid x +. str +Ġinn ov +Ġde ck +' >Ċ +ap ers +] ( +cont inue +s pec +ĠR oad +AS H +ili ar +Ġcontin ues +Ġapp oint +Ġ# Ċ +ĠV ir +Ġ?> " +Ġb in +} ", +go ing +e ach +B D +ĠA ccess +D oc +ĠMan agement +B ER +ask et +.get Instance +Ġestablish ed +so cket +IN S +ĉv irtual +ĉ result +RE AD +_ height +ĠF ont +Ġ( );Ċ +_ html +Ġneighb or +l or +Ġg ather +Ġ} )ĊĊ +Ġid entity +Ġf ab +p adding +ĠR oute +Enumer able +à ´ +Ġfor ced +/j query +.ĊĊ ĊĊĊĊ +res ents +_ left +.P aram +ĉ throw +ĠH am +Ġevent ually +ac er +p ub +Ġtr a +un ique +d el +ĠFlor ida +ĠC lean +x a +Ġ · +Ġvalid ate +Vis ual +Ex pression +_f unc +m ember +ĉ h +tr l +ĉ G +nap shot +ĠProp Types +v in +] )ĊĊ +ow l +if ies +Ġ$ ('. +ĠCont ext +ĠTo ast +. Key +Ġoffic ers +/ n +s n +und efined +. items +ut ow +am age +Ġaccount s +ook ie +Se ction +ici ans +Ġad vis +( is +[: , +ĠFr ance +F unc +ic ious +Ġto k +Ch annel +ĠA D +_N UM +Ġtime out +lem ma +rem e +u j +.A l +uc lear +( os +(" < +[ Ċ +f etch +Ġb al +Ġgu id +- align +ĠW rite +ĠOn ce +utow ired +OD ULE +Ġp itch +C F +by tes +ĠCom mission +Ġincre d +P ER +_ response +ĠL os +par ser +Ġass ume +. Request +ĠT oken +_p osition +Ġn om +- term +Ġrem aining +i ostream +Ġpie ces +ap y +ĠL ess +r ange +umb n +pr ise +_ option +Im pl +k wargs +Ġbusiness es +Al ert +Ġpart ies +ĠCont ainer +ĠPr ivate +ĠPl an +Ġregister ed +Ġj our +ack er +ен и +/ > +ch at +se ct +Ġcre ation +olut ely +Ġinst ant +Ġdel ivery +ick en +y es +ĠFr anc +bl ing +end a +[ ( +_r ange +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠ +Ġsched ule +Con n +Ġthan k +x d +Ġh ook +Ġdocument ation +Param eters +H ello +v t +Ġart icles +Ġw est +def ined +. select +ok ens +ĠV AL +.f ile +res et +Ġmy s +ĠM A +] ), +Ġc ities +rel ated +å Ľ +Ġappe ared +Ġw id +.p anel +ĠIn s +. entity +Ġde cre +ĠL ou +(t ime +ĠTh ank +.create Element +Ġmention ed +oun ce +ĠT ry +ĠW all +/ images +ĠM enu +' čĊ +ĠE r +Ġcrit ic +ĠY ear +( param +Ġf lo +N N +oot er +Ġ ];Ċ +ĠA ff +" github +room s +Ġh yp +g lobal +Ġa vec +æľ Ī +Ġcomplet ion +Ġcon d +onym ous +( temp +Ġst ars +Ġre levant +Ġcover ed +Ġel im +_t ypes +( bool +Ġt u +_ex ists +Ġsec ure +Ġst ored +] / +x F +ĠCont roller +Ġm igr +M I +ĠD en +Ġann ual +U IL +- and +Ġcr ime +b el +Ġk itchen +@ g +_p h +ourn ament +ĠS ocial +ĠS pecial +log ger +Ġt ail +Ġun known +d ed +Ġapp rec +(d b +c f +Ġass ign +- out +ĠM ont +d p +w idget +Ġst one +- primary +. grid +Result s +az z +Ġda ughter +Ġcur r +Ġl in +Ġs outh +form s +ĠO UT +let te +ak s +ig ure +ĠE U +var iable +Ġb rief +ĠSc ott +Ġcon ference +and a +_ lock +or al +Ġe ine +OR S +//////////////////////////////// //////////////////////////////// +ess o +Ġr is +Ġg ender +est ic +L icense +( out +Ġm s +Se e +Ġwill ing +az e +Ġs ports +Ġy es +l u +Ġp urs +/j avascript +- pro +nav bar +_pro duct +/ bootstrap +Ġdr iving +Ġ Ä +Ġpro pos +ult ip +up lic +. email +Ġappro x +( cl +Ġwe ar +Ġrep ly +ass et +Ġ ice +Ġt x +k r +ĠGerman y +ĠGe orge +Ġc b +ĉ err +M ove +Ġpol y +vo ice +} " +Ġan imal +A v +ĠL ocation +Ġn ative +] [" +< double +Ġm ais +, int +Ġpre par +Ġinter val +plement ation +_ ERR +Ġb ug +> " +st at +Ġ} ,čĊ +< span +Ġfa ith +Ġ rom +pre v +ĠE lect +F ind +Ġg od +ot or +// ---------------------------------------------------------------- +orig inal +C pp +ĠSen ate +Ġposition s +Ġweap ons +Ġco ff +Ġpur poses +p ol +Ġim press +Ġanim als +. Entity +(n p +Ġmur der +Ġ` ` +fl ag +Ġsol utions +ĠAct ive +Ġb right +.d ate +Ġsit u +ï¼ Ī +. ID +Ġs ie +), čĊ +ak t +S pace +.d at +.index Of +h an +az ine +ĠZ e +Ġcr ash +( / +> = +Ð ± +iv a +.Auto Size +ĠL at +_ ext +Initial ize +.reg ister +OP Y +Ġre verse +_d is +'] [ +Ġprom pt +ont o +ĠJ ournal +r outer +Ġmys qli +# else +) " +-x s +let s +ph an +. LE +W ill +Ġaff ord +Ġsk ill +-t oggle +N C +B ind +T S +J ust +iter al +Y P +ĉ unsigned +Ġw ind +)) :Ċ +Ġw arning +ĠW ater +Ġd raft +Ġc m +Ġs am +Ġhold ing +z ip +ĠSc ience +Ġsup posed +G en +Ġdi et +< h +ĠP ass +v i +Ġhus band +� � +n ote +ĠAb out +ĠIn stitute +Ġcl imate +.Form at +Ġn ut +est ed +Ġapp arent +Ġhold s +f i +new s +C M +v ideo +': ' +D ITION +p ing +Ġsen ior +w a +-- >Ċ +_ default +ĠD atabase +re p +E SS +ner gy +.F ind +_m ask +Ġr ise +Ġk ernel +:: $ +. Q +Ġoffer ing +de cl +ĠC S +Ġlist ed +Ġmost ly +eng er +Ġblock s +ol o +Ġgover ning +\ F +Ġcon cent +.get Text +Ġm b +Ġocc urred +Ġchang ing +Sc ene +_C ODE +B eh +" The +Ġt ile +ĠAssoci ation +ĉ P +al ty +_ ad +od ies +i ated +Ġpre pared +poss ible +Ġm ort +TE ST +Ġign ore +Ġcal c +Ġr s +Ġassert Equals +Ġs z +ĠTH IS +. "Ċ +Ġcan vas +j ava +Ġd ut +VAL ID +.s ql +. input +Ġa ux +S up +Ġart ist +V ec +_T IME +.string ify +et ween +ĠC ategory +Ġ[ - +ĠDev Express +ĠJ ul +Ġr ing +. ed +Y Y +L et +Text Field +Ġfl at +_p rint +ĠOT HER +ad ian +Ġcheck ed +e le +Al ign +stand ing +Ġ[ ], +Ġl ab +uck y +ĠChrist mas +( image +.m odule +Ġl ots +Ġslight ly +(f inal +er ge +è ¿ +ĠPol ice +ĠR ight +Ġaw ard +ĠO S +Ġ{ }ĊĊ +Ġp tr +ov es +ic ated +еР¼ +Ġman age +olid ay +Am ount +ool Strip +t body +N av +w rap +B B +Ġwatch ing +ari os +Ġoption al +_ K +ĠL icensed +.M ap +T imer +ĠA P +ĠRe v +( o +, c +um in +eta iled +ĠH y +Ġbl ank +ag ger +ĠS elf +() [ +.m ake +ear n +ch annel +< pre +ble m +_p assword +_s p +ic ing +e z +Ġthe ory +ĠT er +, n +log o +ĠHT TP +() )) +.h andle +> ;Ċ +W orld +Ġpy thon +Ġl if +Ġtr av +Ġcon ven +com pany +ĠCl ub +V er +B tn +Ġz one +product s +ĠE duc +Ġver ify +ĠM il +on o +] );ĊĊ +EN CE +Ġpack et +Ġc er +Ġen umer +Ġpar s +form ed +Ġocc up +t re +Ġexerc ise +D ay +_s um +Ġask ing +apt ion +Ġord ers +Ġsp ending +ĠE RR +.D is +ĠU til +âĢľ I +\ ' +? ) +/ >Ċ +Ġem ot +Ġinflu ence +ĠAfr ica +att ers +Ù ħ +.s ession +Ġch ief +ĉĉĉĉĉĉĉĉ ĉĉĉ +Ġto m +clud ed +ser ial +_h andler +.T ype +ap ed +Ġpolic ies +- ex +- tr +bl ank +mer ce +Ġcover age +Ġr c +_m atrix +_ box +Ġcharg es +ĠB oston +P e +Ġcirc um +Ġfil led +Ġn orth +icture Box +ĉ res +è ® +Ġter min +Ġ[ âĢ¦ +IRE CT +Ġb er +Ġ" ../../ +ret ch +.c ode +_c ol +ĠGovern ment +Ġarg v +ĠL ord +as i +Ex ec +ĉ let +vert is +Ġdiscuss ion +en ance +out ube +type of +Ġs erved +ĠP ut +ĉ x +Ġs weet +B efore +ateg y +. of +ĠM aterial +S ort +ON T +ig ital +Wh y +Ġs ust +Ġ ç +ab et +Ġseg ment +Ġ[ ],Ċ +ĠMus lim +Ġfind ViewById +c ut +_T EXT +ĠM ary +Ġlo ved +Ġl ie +ĠJ O +Ġis set +mon th +Ġpr ime +t i +ĠCar ol +U se +ĠP op +ĠS ave +Int erval +ex ecute +d y +ĠI ran +_ cont +ĉ T +Ġph ase +check box +we ek +Ġh ide +Ġt il +Ġj u +C ustom +b urg +/ M +T ON +Ġqu ant +Ġr ub +ix els +Ġinst alled +Ġd ump +Ġproper ly +( List +Ġdec ide +app ly +H as +Ġkeep ing +Ġcitiz ens +Ġj oint +p ool +S ocket +_ op +Ġweap on +gn ore +ĠEx ec +ott en +ĠM S +Ġ( - +ĠRe view +Ġex amples +Ġt ight +! ( +D P +ĠMessage Box +Ġphot ograph +UR I +é t +l ow +ĠGr and +.p ersistence +Ġmaint ain +Ġnum s +Ġz ip +ial s +ĠG ets +pe g +ĠB uffer +~~ ~~ +ra structure +ĠP L +u en +ob by +size of +Ġp ic +Ġse ed +Ġexperi enced +Ġo dd +Ġk ick +Ġproced ure +avig ator +- on +, j +ĠAl though +Ġuser Id +ac cept +Bl ue +IC olor +l ayer +av ailable +Ġend s +.t able +Ġdat aset +b us +Ġexpl ain +( pro +ĠCommit tee +Ġnot ed +] :Ċ +D im +std io +. ",Ċ +_s ource +ĠWe ek +ĠEd ge +Ġoper ating +Ġest e +i pl +ag ination +Ġpro ceed +Ġanim ation +.Model s +ĠW atch +i at +Ġopp on +/ A +Re port +Ġs ounds +_b uf +IEL D +Ġbu nd +ĉ get +.p r +(t mp +Ġk id +>ĊĊ Ċ +Ġy ang +Not Found +Ñ Ĩ +m ath +@g mail +ĠL IMIT +red ients +Ġv ent +avig ate +L ook +Ġrelig ious +Ġr and +ri o +( GL +_ ip +u an +ici ency +ĠCh ange +> čĊčĊ +ĠEnt ity +Ġrencont re +ĠR et +pl an +é n +BO OL +ur ies +tr ain +Def inition +======== ==== +z z +An imation +ĠO K +_m enu +.b l +_s core +Ġac ad +( System +Ġref resh +'=> $ +.G raphics +ament o +p id +t c +Ġt ips +Ġhom es +Ġf uel +â ĸ +_h elper +ĠĠ čĊ +ĠR oom +.C lose +_ attr +ĠM ount +ĠE v +ar ser +_t op +e ah +ĠDe lete +ãĢ į +u ke +Ġus age +ar ia +_de v +Ġtext ure +Ġconvers ation +e per +Be an +d one +non atomic +ĠSe cond +Ġshoot ing +_p re +Com ponents +Ġ] ĊĊ +__ , +stit ution +.Ch ar +> ();ĊĊ +Ġpresent ed +Ġw a +ok er +- ĊĊ +in er +Ġbe coming +Ġinc ident +At t +Ġreve aled +for c +Ġbo ot +.p age +Enumer ator +_ -> +Ph oto +Ġs pring +. ", +ĠD ictionary +B JECT +Ġloc ations +Ġs amples +Input Stream +ĠB rown +Ġst ats +qual ity +Ñ ħ +-d is +Ġhelp ing +Ġp ed +( se +ĠWh o +al ian +int ernal +Ġf t +> (). +-> { +Ġm ine +Ġs ector +Ġg ro +Ġopport unities +Ġà ¼ +Ġm p +Ġalleg ed +Ġdoub t +M ouse +Ab out +_p art +Ġch air +Ġstop ped +lo op +ent ities +Ġapp s +ans ion +Ġm ental +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠ +F R +Ġdef end +c are +Ġide al +/ api +ur face +Ġe le +ul ator +ĠR ights +angu ages +Ġfund s +Ġad apt +At tributes +Ġdep loy +opt s +Ġvalid ation +Ġconcern s +u ce +.n um +ult ure +il a +Ġc up +Ġp ure +.F ore +ĠHash Map +.value Of +as m +M O +Ġc s +Ġst ores +Ġ ************************************************************************ +Ġcommunic ation +m em +.Event Handler +. Status +_ right +.set On +S heet +Ġident ify +ener ated +order ed +Ġ" [ +Ġs we +Con dition +ĠA ccording +Ġpre pare +Ġro b +P ool +Ġs port +r v +ĠR outer +Ġaltern ative +( [] +ĠCh icago +ip her +is che +ĠDirect or +k l +ĠW il +key s +Ġmy sql +Ġw elcome +k ing +ĠMan ager +Ġca ught +) }Ċ +S core +_P R +Ġsur vey +h ab +He aders +AD ER +Ġdec or +Ġturn s +Ġr adius +err upt +C or +Ġm el +Ġin tr +( q +ĠA C +am os +M AX +ĠG rid +ĠJes us +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠ +.D E +Ġt s +Ġlink ed +f ree +ĠQ t +Ġ/** čĊ +Ġf aster +ct r +_ J +D T +.C heck +Ġcomb ination +Ġint ended +- the +- type +ect ors +am i +ut ing +Ġum a +X ML +U CT +A p +ĠR andom +Ġr an +.s ort +Ġsort ed +. Un +_P ER +it ory +Ġprior ity +ĠG al +ĠO ld +h ot +ĠD isplay +(s ub +_T H +_ Y +ĠC are +load ing +K ind +_h andle +, , +r ase +_re place +.add EventListener +ĠR T +Ġenter ed +g ers +Ġ ich +( start +/ app +Ġbro ther +M emory +Out let +Ġ utf +pre c +Ġn avigation +OR K +Ġd st +D etail +Ġaud ience +Ġd ur +Ġcl uster +un ched +Ġ ], +Ġcomfort able +. values +ĠT otal +Ġsn ap +Ġstand ards +Ġperform ed +h and +(" @ +å Ń +Ġph il +ib r +tr im +Ġfor get +Ġdo ctor +.Text Box +icon s +, s +ĠO p +S m +St op +ĉ List +ĉ u +Com ment +_V ERSION +.X tra +P erson +r b +LO B +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +ĠCent ral +IC K +ra q +Ġput ting +Ġm d +ĠL ove +Pro gram +B order +o or +Ġallow ing +a fter +Ġent ries +ĠMay be +] ). +ĠSh ort +) \ +.n ow +f riend +Ġpre fer +ĠG PIO +os is +ĠGame Object +Ġsk ip +Ġcompet ition +_m atch +lic ations +_CON T +.group Box +Ġal s +" We +_e q +l an +_ search +ĠMus ic +as is +Ġb ind +ĠIs land +r um +( E +Ġse at +V ideo +Ġa ck +ree k +={ () +Ġr ating +Ġrestaur ant +DE X +(b uf +pp ing +ual ity +Ġle ague +Ġfoc used +ap on +$ data +CL UD +CLUD ING +Ġabs olute +( query +Ġtell s +A ng +Ġcomm unities +Ġhon est +ok ing +Ġap art +ar ity +/ $ +_m odule +ĠE nc +. an +.Con fig +C re +Ġsh ock +ĠAr ab +I ENT +/ re +Ġre trie +ycl er +is a +ĠO rgan +. graph +Ġ í +ĠB AS +En um +Ġposs ibly +ÑĢ аР+ĠJapan ese +Ġc raft +ĠPl ace +Ġtal ent +Ġfund ing +Ġconf irmed +Ġc ycle +/ x +G E +Ġhe aring +Ġpl ants +Ġm outh +p ages +or ia +ĠRem ove +_t otal +Ġo d +oll apse +do or +Ġb ought +Ġadd r +AR CH +_d im +dd en +Ġdec ades +RE QUEST +Ġvers ions +f ire +Ġmov es +f b +Ġcoff ee +.con nect +ĠR ow +Ġs chema +S cope +- Type +Ġfight ing +Ġret ail +Ġmod ified +T F +File s +n ie +_com mand +st one +Ġ ÑĤ +_ thread +Ġb ond +ĠDevelop ment +Ġp t +F ORM +ple t +Ġident ified +c pp +Ġc oding +ok ed +ĠM aster +ID TH +Ġres idents +red it +ĠPh oto += - +un te +ate ur +_ST ATE +ĠS ing +Ġshe et +. val +or se +Ġh ers +Ġdetermin ed +Com mon +Ġw ed +_ queue +P H +ĠAt l +cre d +/L ICENSE +Ġm es +Ġadv anced +.j ava +.S h +G o +k ill +f p +_set tings +Ġp al +Ġtr uck +Ġcomb ined +Ġ" ${ +ĠCor por +Ġjo ined +ĠJ ose +ĠC up +un s +est ival +lev ision +Ġbro ken +Ġmar riage +ĠWest ern +Ġrep resents +ĠT itle +Ġs s +.A ss +ongo ose +ient o +< >();Ċ +Ġabs olutely +Ġsm ooth +TER N +ĠUn less +W ord +Ġmer ge +ig an +ĠV ol +Ġn n +.get Id +ĠÐ · +Ġsex y +Ġseek ing +S ingle +. this +Ġk om +b ound +; " +Ġfont Size +_d f +Ġinj ury +( H +Ġiss ued +_ END +: self +Ġp atch +Ġle aves +Ġad opt +File Name +ãĢ IJ +Ġexec utive +ĠBy te +] ))Ċ +Ġn u +out ing +clud ing +- R +. options +Ġsub stant +av ax +ĠB UT +Ġtechn ical +Ġtw ice +Ġm ás +Ġun ivers +y r +Ġdr ag +ĠD C +Ġs ed +Ġb ot +ĠP al +ĠH all +forc ement +Ġa uch +.m od +not ation +_file s +.l ine +_fl ag +[ name +Ġres olution +Ġb ott +(" [ +end e +( arr +F ree +( @" +ĠD istrict +PE C +: - +P icker +ĠJ o +ĠĠĠĠĠ Ċ +ĠR iver +_ rows +Ġhelp ful +Ġmass ive +--- Ċ +Ġmeas ures +ĠR untime +Ġwor ry +ĠS pec +ĉ D +ãĢ ij +Ġ) {Ċ +Ġwor se +(f ilename +Ġl ay +Ġmag ic +ĠThe ir +ou l +st roy +ĠWh ere +Ġsu dden +Ġdef e +Ġb inding +Ġfl ight +ĠOn Init +ĠW omen +ĠPol icy +Ġdrug s +ish ing +(' ../ +ĠM el +pe at +t or +Ġpro posed +Ġst ated +_RE S +Ġe ast +ĠCON DITION +_d esc +Ġwin ning +fol io +M apper +ĠP an +ĠAn ge +.s ervlet +Ġcop ies +L M +Ġv m +å į +Ġd ictionary +S eg +el ines +ĠS end +Ġ iron +ĠF ort +.d omain +Ġdeb ate +Not Null +e q +ach er +l f +ĉf mt +Ġlaw y +Ä Ł +ĠM en +Ġtr im +( NULL +Ġ! ! +Ġp ad +Ġfollow s +"] [" +re qu +ĠE p +.g ithub +( img +et o +(' \ +S ervices +umbn ail +_m ain +ple ted +fort unately +Ġw indows +Ġpl ane +ĠCon nection +. local +u ard +} \ +== " +and on +ĠR oy +w est +ig inal +em ies +it z +') :Ċ +ĠP eter +Ġt ough +Ġredu ced +Ġcalcul ate +Ġrap id +c ustomer +Ġeff icient +Ġmed ium +Ġf ell +. ref +ĠC as +Ġfeed back +S peed +( output +aj e +Ġc ategories +Ġfe e +} ; +Ġde leted +re h +Ġpro of +D esc +B uild +Ġs ides +.Array List +- % +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ø ± +.m atch +л и +Ġfe els +Ġachie ve +Ġcl im +_ ON +ĠC D +Ġteach er +_c urrent +b n +_P L +ist ing +En able +G EN +Ġt v +Ġso ck +Ġpl ays +Ġdis count +ĠK E +ĠDe bug +F ore +ĠI raq +Ġappear ance +M on +Ġst yled +ĠH uman +i ot +ĠH istory +Ġs ac +ĠC ollection +Ġrecomm ended +.Se lected +Ġorgan izations +Ġdiscover ed +co hol +ad as +ĠThom as +M ay +Ġcons erv +Ġdom in +ĠF ollow +ĠSe ction +ĠTh anks +User name +Ġrec ipe +Ġwonder ful +.s leep +_ if +ĉĊ ĉĊ +orn o +Ġr u +_t arget +." " +à ¦ +Event Args +Ġinput s +Ġf if +Ġv ision +c y +ĠS eries +) ((( +Ġtr ading +Ġmark er +B egin +Ġtyp ically +Ġca uses +drop down +_DE BUG +Ġdet ect +c ountry +! ");Ċ +ĉ R +app y +Ġc ref +(' < +" => +ĠL E +read er +Ġadmin istr +à µ +uck et +Ġf ashion +. char +iz ar +Ġdis able +Ġsu c +ĠL ive +iss ue +Ġmet adata +fl ags +Ġ ðŁ +Ġcomm itted +Ġv a +Ġr ough +Ġ'' 'Ċ +Ġhigh light +_var s +V O +Ġenc oding +- Z +_s ign +$ ("# +Ġr ain +reate st +ĠEN D +Se lection +Ġcandid ates +Ġs av +. Empty +Ġdec isions +Ġcoll abor +rid ge +fe ed +ress ion +Ġperson s +V M +eg a +_B IT +A ccording +ack ed +Ġdoll ars +_lo ss +ĠC ost +} "Ċ +Not ification +Ġpro stit +Ġauthor ity +.re c +Ġsp okes +ĠT oday +ist ant +ĠHe ad +âĢĿ . +ertain ment +ce an +cul ate +Ġv en +How ever +_ arr +Ġtok ens +G raph +ĠJ ud +ĠVir gin +ĠS erial +un ning +M utable +ag ers +.c sv +Ġdevelop ing +Ġinstruction s +Ġprom ise +Ġrequest ed +_ encode +/ " +ĠI con +u ilt +- day +Ġint elligence +. IS +ĠO bservable +ĠH ard +Bo ol +ident ial +.An chor +Ġsell ing +C I +AG ES +t le +b ur +UFF ER +R Y +Ġbig ger +Ġr at +Ġfam ous +Ġtyp ename +Ġexpl ained +} }Ċ +Ġn uclear +- N +Ġcr isis +ĠEnt er +Ġan swers +/ ${ +/ pl +Ġse qu +_n ext +m ask +Ġstand ing +Ġpl enty +ĠC ross +ĉ ret +d ro +ĠC ast += true +ĠCh ris +ic io +ĠM ike +Dec imal +add Component +L en +Ġco ck +Ġ# { +UR N +< tr +Ġauthor ities +Res ources +- H +B ottom +_ qu +put er +ester day +Dis patch +s ince +Ġfam iliar +, i +V C +Ġm ent +, C +Ġfre edom +Ġr outes +ĠB uy +Ġcomm ands +Ġm esh +/ C +ĠSet tings +- style +Ġw itness +Ġc le +Ġun ion +ef ault +are t +Ġthought s +Ġ ---- +_pro cess +_ us +ing ly +U ES +T ouch +ĠÐ ¼ +_ open +ĠV ec +Ġre ward +.C lick +/ : +Ġn ie +Ch anges +M onth +ï¼ Ł +Ġexec ution +Ġbe ach +( Integer +ĉ a +/ ' +.Font Style +Ġab ort +ĠS ingle +( isset +Ġd p +Ġ}} +Ġ* = +ĠP S +Ġdanger ous +[ p +OM E +O ther +ĠString Builder +Point s +head ing +Ġc urrency +Ġpercent age +_A PI +Ġclass ic +the ad +ĠM O +F E +Id x +aw ait +Ġà ¨ +Ġacc ident +Ġvari ant +Ġm yst +ĠL and +ĠB re +Ġh arm +ĠA cc +Ġcharg ed +ion es +Vis ibility +ar ry +ĠL anguage +Ġwalk ing +" .ĊĊ +if er +Ġleaders hip +.F rom +yn am +Ġt imestamp +i pt +ĠH as +REF ER +ĠIt s +Ġlist ener +UT E +_d escription +Ġexperi ences +Ġcre ates +R S +c art +bl ack +Ġcho ices +w ar +Ġ'' ' +Ġorder ed +Ġeven ing +Ġp il +Ġt un +ĠB ad +( app +r andom +Ġexp licit +Ġarr ived +Ġf ly +Ġecon om +-m ail +Ġlist s +Ġarch itect +ĠP ay +Ġd s +ĠS ol +Ġveh icles +H z +- com +Ġk ing +_e qual +ĠH elp +Ġab use +-- ;Ċ +Ġex tr +Ġchem ical +ä ¿ +Ġor ient +Ġbre ath +ĠS pace +(e lement +w ait +DE D +ig ma +Ġent r +Ġs ob +- name +Ġaff ected +ik a +Ġco al +_w ork +Ġhundred s +Ġpolit ics +sub ject +Ġconsum er +ANG E +Ġrepe ated +S end +Ġ# [ +Ġprot ocol +Ġlead s +use um +E very +Im port +(c ount +Ġchalleng es +Ġnov el +Ġdep art +b its +.C urrent +Ġ` ${ +ot ing +( \ +Ġcreat ive +Ġbu ff +Ġintrodu ced +us ic +mod ules +A re +-d oc +l anguage +_c ache +Ġto d +? > {{ +ĠRes ource +ĠSt andard +ĠP rem +up dated +ival ent +Ġas sets +_t emp +Ġinterest s +Ġhard ware +ĠR om +ĠSh are +Ġ' 'Ċ +Ġ* , +ĠT ake +ĠIm ages +_C HECK +(type of +ĠJ un +\< ^ +Ġli qu +Ġwor st +ymb ols +ĉĉĉ ĠĠĠ +Ġdr ivers +ĠD ocument +en o +ĠTechn ology +Ġappro ved +ump s +Ġs now +form ance +_A SSERT +u its +Ù Ĩ +Ġdiffer ences +. Visible +ĉĉĉ čĊ +ĠP s +_f etch +Ġto do +. ',Ċ +Ġs el +ur ers +in valid +Ġt weet +V EL +Ġresearch ers +Ġs printf +ĠR O +Ġp el +.Tr ans +Ġil legal +d ialog +sm arty +l g +_M IN +Ġher o +f inal +Ġp p +.L e +Ġc i +ĉ RT +Ġsuggest ed +p df +ach ing +ĠR o +ĠProp erties +ĠS i +Ġbuy ing +Ġm u +Ġl ands +if iers +ĠF ILE +RO UP +Ġh older +ĠS on +Ġsym pt +.r oute +) ? +Ġarg c +Ġfor t +Ġcas ino +_c ategory +Ġfor um +p refix +apt ure +T ube +em s +im ize +Ġn ue +a us +c ourse +AT OR +() ), +Ad vertis +ING S +Ġack now +ĠKore a +pl ing +Ġwork er +PL IED +h al +ĠRich ard +Element s +ĉĉĉ Ġ +st ar +Ġrelationship s +Ġche ap +AC H +ĠX ML +, & +ĠLou is +Ġr ide +_F AIL +Ġch unk +[ s +_O UT +Ġch osen +_ [ +/ ( +ĠJ eff +_s l +pr iv +ĠCan adian +Ġun able +_F LAG +Ġn os +h igh +Ġl ift +f un +() { +el ly +ycler View +_ as +_L IST +Ġr adi +.get Value +ĠAnge les +ĠS pan +_in stance +it ors +Ġm igration +A K +O h + ® +. selected +ĠG T +Ġadv ance +ĠSt yle +.Data GridView +e ction +Ñ İ +p io +ro g +Ġsh opping +ĠR ect +I lluminate +O U +ĉ array +Ġsubstant ial +Ġpre gn +Ġprom ote +IE W +.L ayout +Ġsign s +/ . +Ġlet ters +Bo ard +ct rl +" \ +ĠJ ones +Ġvert ex +Ġj a +Ġaff ili +Ġwe alth +ĉ default +Ġsignificant ly +Ġe c +Ġx s +act ual +.p er +_st ep +an vas +m ac +Ġtrans l +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Iter ator +Ġo ch +agnost ic +ĠD uring +ĠDE FAULT +Ġt ill +Ġsign ature +Ġb ird +ĠO l +ĠI r +H S +av atar +ESS AGE +Ġe lev +Ġm t +ĠN av +Ġrel ax +Ġpl ate +IT EM +( date +.n ot +Ġgr ade +Ġ} ),Ċ +? "ĊĊ +i ences +H igh +ĠD IS +dis abled +Q UI +Ġno ise +a ux +ĠU P +os a +Ġv oc +Ġ )) +oc om +_O FF +ĠD b +L ock +.e clipse +, d +ĠD raw +Ġ" ( +Ġvis ited +Ġâ Ī +Ġsuc ceed +Ġim possible +a ire +ĠT urn +Ġd ish +F G +Ġs ensor +AN N +ab a +Ġsur g +] );čĊ +Ġf p +_ an +- J +- G +ĠJ ob +Con vert +ĠKE Y +Ġauth ors +_s erver +\ r +Ġ-* - +f lex +Ġs oc +R et +Ġs alt +ĠâĢ¦ ĊĊ +ĠC lear +(p age +-d anger +Ġroom s +con v +# { +. op +ĠA rea +_S C +h en +Ġbeg ins +- y +Ġexc ited +Ġign ored +Ġbon us +st udent +ĠM ember +Ġrel atively +ĠL ow +ĠPro du +ate way +pos ure +Ġth ick +ani el +( view +ĠCr ush +Ext ension +I l +e ed +LO C +. im +. Items +Ġconflic t +.pre vent +Ġon Create +u v +is er +Ġw ave +M ar +ĠComm unity +ic he +ĠNo thing +[ m +ĠLe e +ri ends +è re +!! ! +an z +. result +ĠS K +_P ARAM +Ġdem ocr +Back Color +.ex ists +" It +( options +ra zy +as er +\ Database +al endar +_ ass +; }Ċ +vert ex +ine craft +W arning +arg o +Ġact or +ĠInst ead +ĠUs ing +S elf +@ interface +Ġspe aking +ĠPar is +ĠL ICENSE +.n ode +ĠF ood +E IF +ĠB i +. Start +ĠI B +Ġun iversity +ĠHe ader +.pro duct +C opy +et c +r ical +Ġ> >> +book s +Ġal gorithm +Ġ' __ +(j avax +Ġnumer ous +Sh are +H ave +Ġrec ru +Ġpro ve +.sub string +he alth +е л +Ġdec imal +Ġcomm ission +s cription +x C +Ġsum mary +att ed +Ġclo ser +fin ished +() ){Ċ +ĠW ood +_field s +k u +_ items +Fl ag +Ġconf idence +ĠF ederal +du x +Ġcomp at +Ġvert ical +Ð ¹ +è s +; ">Ċ +_m anager +() ))Ċ +ID E +: ", +__ Ċ +ĠW ay +Ñ Ī +T emp +ĠS TR +rit ten +S ync +ĠA V +ĠC EO +ĠG uid +Ġenvironment al +Ġcorrespond ing +ĉ console +Ġjust ice +ĠJ S +Ġl ived +g ar +ĠG raph +ĠSt at +Ġi Phone +. al +ĠH D +Ġocc ur +Ġth reshold +Ġon click +RE G +.Graphics Unit +M eta +Å ¾ +Ġc um +.g nu +à « +Ġobt ained +Ġcompl aint +Ġe ating +Ġt ar +_t ask +Ġopt s +( to +P ass +Ġpl astic +t ility +ĠW in +.prevent Default +p ile +ĠG ar +Ġqu antity +_l ast +Ġg reatest +D ao +_D IS +ĠUs ed +ĠH P +rit ing +S ION +bl ue +d omain +Ġs cores +N ormal +_ admin +ĠA SSERT +Th en +** * +d ist +l on +Ġh ate +sh al +Image View +d atabase +Ġp and +Ġlog ic += false +b g +ĠConfig uration +Ġn ur +O G +Ġmar ried +: + +Ġdro pped +Ġreg istration +оР¼ +ult iple +iz ers +sh ape +.c opy +Ġwe aring +ĠC ath +Ġded icated +Ġ.. .Ċ +Ġadv oc +ĠF amily +Ġstat ements +em atic +ampions hip +Ġmot iv +ĠH ave +Ġbl ow +J ob +c ert +_v ector +inst all +ĠC OPY +em bed +D IR +ĠS pring +Ġex hib +cd n +ĠCom ment +ĠOption al +. player +ĠD ark +( pos +ĠSh ould +Ġcent re +ĠGu ard +ó w +Ġtr ouble +EN ER +( unsigned +_s ervice +Ġn s +ul ing +ĠMex ico +ĠN Y +mys ql +Ġl ic +å ľ +M r +- fl +ĠC ustomer +id i +Ġ? >ĊĊ +ri ble +Ġп ÑĢ +Ġs izes +_STR ING +valid ation +ĠJ on +( Http +add Class +N odes +Ġfrag ment +Ġsp oke +Ġw aste +J oin +Ġill ustr +el i +c ient +Ġa id +Ġpro sec +') {Ċ +Ġpass ing +Ġf aces +Sh ape +_ Z +it i +Ġal le +Ġro bot +ĠĠĠĠĠĠĠ Ċ +ĠS pe +Ġrece iving +ĠD etails +Ġ" ) +m g +_RE F +Ġcompar ison +* , +ĠF ound +_s ession +( U +/ F +Ġx xx +N etwork +d ers +Ġcap ture +Ġcor re +ĠL td +ĠAd v +[ @ +Ġcl ip +M ill +ĠPro file +Ġend if +Ġob lig +des cribe +.e lement +riter ion +L D +er ed +Ġfav our +s core +ĠF ilter +at tributes +Ġcheck s +In flater +ĠPl us +Ġscient ific +Ġpriv acy +He ad +Ġfe at +Ġdeg rees +ĠP ale +; "> +Ġfil ms +ĠA udio +ĠT ag +ĠE nergy +it ar +par ator +Ġf ellow +Ġev t +ĠT ri +ĠD AM +cl oud +ĠP assword +ĠDemocr ats +ĠAc ad +$ lang +Ġre b +() )ĊĊ +н Ñĭ +ĠB ur +read cr +Ġh ex +Con sole +ct l +ous el +ĠWill iam +Ġa z +_P ORT +Ġpract ices +Ġany where +ĠP osition +Ġ- >Ċ +i ams +.user name +place holder +Ġo der +ĠSecret ary +Ġi T +mon d +event s +? âĢĿ +.S ub +Ġatt ached +Ġn ão +Ġest ate +. action +Ġfig ures +Ġ} );čĊ +Ġsubs cri +.t ag +n am +. plot +no on +li ament +Char acter +.t ab +Ġw inter +ĠVar iable +Ġtre es +Ġpr oud +( V +_ load +Ġh ier +ĠE con +Ġf d +Ġvict ims +R est +ian a +Ġf ake +.Print ln +Ġstr len +Ġs ad +Ġb le +Pro t +Ġbutton s +Ġte levision +Ġlog o +ext ension +ĉ j +ste in +acion es +Ġ"" "ĊĊ +Ġsim p +Ġrecord ed +Ġbr ings +Ġprincip al +Ġfe es +(s ource +k dir +Ġutil s +Ġcorrect ly +f il +Ġw el +P air +-b utton +s cale +ver ify +[ c +Ġ-- - +Ġes cape +ik es +Lower Case +ic ian +Ġch apter +ĠT YPE +Ġsh adow +Ġaw esome +W E +el if +Ġl ambda +Ġdist inct +Ġb are +- off +Ġcol our +.append Child +ole c +ag a +.f ill +ĉs uper +Ġad j +( position +.get Item +Sh ort +Ġtot ally +V D +ĠT re +_ ep +v ements +ĠS olution +Ġfund ament +F ollow +Ġfac ility +Ġhappen ing +O F +.text Box +S pan +Ġ « +id en +Ġex ceed +(p arent +Ġc p +ç » +Ġhas n +Ġp ri +Ġcon sequ +n en +ĠIN TO +I gnore +ĠF uture +Ġcar bon +ĠSte el +f mt +ok ie +Ġs pl +(t itle +- info +Ġde als +Ġfix ture +e a +D iv +Ġtest ed +_ return +)ĊĊ ĊĊ +upport ed +ĠC ook +Ġpay ing +ĠI ll +Ġarrest ed +ĠPr ime +_c allback +> ,Ċ +dr iver +On ce +ab b +_by tes +ĠS ets +( Object +Ġc c +Ġsh ell +al o +); // +( log +ct ors +) +Ġ$ (". +.p os +Ġbo ys +Ġwed ding +Ġag ents +=" _ +ĠAr my +Ġh int +v ision +Ġte ch +ĠCon nect +Ġleg end +ĠB et +.B ase +Sub ject +Ġl it +Rem ove +Ġ" : +ĠF inal +pear ance +ĠiT unes +Ġparticip ants +ĠPy thon +Ġbus y +i el +vert ices +Ġtemplate Url +ĠC lose +Im g +ĠCorpor ation +t imestamp +Ġext end +Ġwe bsites +Ġposs ibility +о ÑĤ +Ġk ö +Ġme at +Ġrepresent ation +Ġ ĉĉ +_ST ART +.app ly +ĠVal ley +ĠS uccess +H i +Ġn ob +ĠI Enumerable +_ select +ge o +. ")Ċ +Ġturn ing +Ġfab ric +(" ");Ċ +Ġpers pective +é Ĺ +ĠS n +Th ank +; j +.Param eters +ĉ ĠĠĠĠĠĠĠĠĠĠĠ +Ġfact s +Ġun t +.in stance +################################ ################################ +- end +ĠJO IN +ĠH en +Ġur i +åIJ į +Ġн а +ĠIn fo +Ġconduct ed +Ġà ¥ +OUR CE +Ġw ine +J ohn +.Error f +ĠA ge +ound ed +Ġreal ize +Ġ] ; +Ġsub sequ +, m +( User +ian o +Ġaccom pl +is p +.st d +é ĩ +ĠB ed +.set Attribute +B R +ke ep +ĠA LL +Ġis ol +am ma +P ackage +Ġoccas ion +-s uccess +еР´ +ĠLIMIT ED +st rip +() ĊĊĊ +istrib ution +Color s +Ġ+ :+ +Did Load +al er +Ġt id +ĠL ED +ĠLink ed +ĠC art +() )čĊ +_RE AD +Ġkill ing +ĠP HP +fe ction +Ġinst ances +c v +"/ > +Ġs f +Ġtax es +_ location +ĠBit coin +u able +r ank +ign ore +tr ack +к а +Ġshould n +ĠO P +=> {Ċ +Ġk m +Ġh elper +_ head +ĠWh ether +oc o +_b l +Ġstat istics +Ġbeaut y +Ġto g +t ip +ëĭ ¤ +Ġc sv +(s ql +std lib +we ak +Ġlik es +Ä į +Ġrepe at +Ġap artment +Ġem ph +_ edit +Ġv it +ĉ type +E ven +ut en +Ġcircum stances +b ian +Ġs ugar +W indows +ì ŀ +Ġobs erved +/ data +Ġcal endar +Ġstri ke +ĠR ES +_s c +f ony +ore m +( z +p ower +et ect +ĠS at +.d escription +Ġg ang +ĠS ports +ong s +ĠB undle +.s um +on ce +Ġacc used +Ġexplo re +Ġapprox imately +Ġlos ing +thes is +ĠF und +Ġdi agn +A utowired +prop erties +Ġ_ . +Ġc nt +ced ure +Ġy y +Ġgr ant +so ck +.inner HTML +Ġ] );Ċ +ĠCON FIG +=' $ +] ];Ċ +UN D +Ġg lob +Ġd ire +uff le +_M EM +Ġauth entic +> (" +Ġdec ade +ĠIm port +Ġorigin ally +Ġj Query +Ġindic ate +Ġours elves +S w +.l bl +ener ate +Ġbas ically +ĠH om +Ġ+ #+ +ĠBrit ain +ĠK ar +to Equal +.st op +Ġmod al +is i +Ġsuggest s +Ġd type +Ġt ur +b f +Ġconnection s +ĠB efore +ist ed +m ouse +Ġpul led +.b uild +Ġlegis lation +Ġfor th +p ad +eg o +.N ow +Ġexc iting +}ĊĊ ĊĊ +Ġcom pr +Ġsh ares +Ġr ig +g reen +_ vec +Ġenumer ate +A uto +ic ator +ĠR ay +as se +Ġh oliday +Ġnull able +g un +_d etails +Ġwr apper +se q +ĠYou ng +ju ana +Ġ" __ +lic ense +ser ve +^ ( +id ers +.Rem ove +rop down +' S +p in +(t oken +.D efault +Ġreason able +amp ion +ĠS ociety +Ġbe i +erv es +r ad +ĠF ox +_ images +Ġw heel +') [ +Ġc fg +( By +Con structor +Ġv ary +.sw ift +Ġpro xy +ĉ H +ĠAn other +ĠP en +Ġcheck ing +Ġj est +man ager +Or igin +ug s +o ir +>< !-- +Ġexpress ed +Ġmod er +Ġag encies +Ġi h +-h idden +ious ly +ĠR od +Ġso le +M ed +.A ny +Ġp c +b al +Ex ample +ĠS ale +Ġst rip +ĠCom p +Ġpresident ial +M ost +put ation +( ref +ĠF our +_f ilename +Ġen forcement +Ø ¯ +ĠGe org +we ights +/ l +Ġag gress +Ġd rawing +and y +< I +- j +ak a +h ref +Ġteach ers +_ Q +( it +ĠM B +Ġtemp orary +ire base +str a +æĹ ¶ +è ´ +( label +ou p +Ġtop ics +Ġport ion +id os +ĠJew ish +Ġre covery +Ġstand s +# [ +Ġafter noon +ĠArt icle +_ att +Ġexpl an +ĠP ak +.setOn ClickListener +. children +Ġi k ++ ( +l ag +Ġdis k +Ġcont rovers +"> & +as p +Ġw ie +ĠAustral ian +ĠYou Tube +At tr +cont ains +du ce +ĠM att +at ern +Ġvol unte +Ġnew sp +V P +olt ip +Ġde legate +_m eta +Ġaccur ate +ĠEx ample +% , +ĠD aily +Ġc abin +ĠS W +Ġlim its +k ip +Ġar my +Ġend ing +Ġb oss +ĠD ialog +Al so +="# " +ord an +row se +- min +Ġ" & +_ loc +U X +Ġdevelop ers +Ġaccur acy +Ġmaint enance +Ġhe av +Ġfil ters +.T oolStrip +Ġn arr +ĠE mp +ORD ER +ĠM obile +.S erial +.out put +.c ol +M aterial +um a +Ġconsum ers +sh ift +Ġp ued +Ġmin i +c ollection +Ġk an +.c enter +H istory +Ġben ch +() ); +itor ies +Ġcrow d +_c all +Ġpow ers +- E +Ġdis miss +Ġtalk s +ĠCh annel +for ward +_ control +/s rc +i est +**************** ******** +Ġbet a +(c olor +_O BJECT +ĠA pi +Ġeffect ively +C amera +s d +uss y +D ict +ĠE ffect +ib ilities +Ġreturn ing +ĠF ar +Ġ' ') +Ġmod ules +il ation +Ġ( % +TR GL +Ġst orm +on na +ĠEX P +Ġs pons +Ġdis pl +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +f all +å Į +ign Key +_ US +et rics +Ġhand les +T L +_ amount +ow a +br and +ĠT ool +Ġus ual +. Z +cre ment +ad ium +st ock +Ġserv ing +ĠB on +Ġline ar +ĠT arget +ĠR adio +H L +Sh ader +om atic +ag ues +in ity +d iff +_ iterator +qu ot +Ġ ,Ċ +c allback +Ġsympt oms +[ _ +ĠB ul +ĠF eb +und o +_ account +Ġtyp edef +и Ñģ +tr as +User Id +ĠP enn +ĠSup reme +} > +user Id +ĠK im +Ġg a +Ġart ists +å ¸ +ĠAb stract +ok emon +Ġh am +o val +Ġch a +at en +å Ĩ +F ixed +Ġvul ner +ĠParam eters +qu antity +.C lear +Servlet Request +Ġy a +Ġsou l +trans action +Ġsol o +Ġp airs +æ Ķ +ĠG re +_ word +ĠC C +Ġg i +z ie +Ġsched uled +rot ation +gy pt +ul ous +:: _ +ĠE ll +< ! +ĉĉ ĠĠ +l p +ah a +C opyright +Ġdr am +Ġdi agram +ĠM em +Ġg arden +Com p +Ġattempt s +uff ix +> () +Ġphil osoph +_re l +å ¼ +Ġs v +.se cond +ant o +.J son +ĠTe le +_ local +_s end +Ġas pects +ì Ĺ +IB LE +Ġr ail +Ġwid ely +ash ed +i ar +in f +up per +d jango +_result s +iss ing +Ġequ ivalent +OUN D +Ġt y +Ġpotential ly +Advertis ement +ĠRec ord +resent ation +_w idget +ound ing +Ġrelig ion +Ġcons c +ĠL im +. am +H tml +Ġ' : +P ATH +_s pec +ort ed +id ades +_sh ape +Ġkeep s +.S ave +ĠL oc +or i +ĠT EST +unic ip +Ġreg ions +Ġbelie ves +/ en +pos ite +{ ' +pre pare +_ const +s ample +ĠWill iams +Ġstr t +_ Get +ĠAnd rew +. active +Ġl ayers +Visual Style +az y +ĠK n +Ġac id +ĠAs ia +Ġex cess +ĉm y +Ġkey board +ens us +Ġcre w +Ġmiss ed +m aster +ĠW ild +Ġnew ly +Ġwin ner +Ġst ub +ic ode +.m ove +D omain +ĠS ar +Ġfore st +LE D +claim er +.ex it +ĠW indow +Ġres istance +ĠC HECK +(" - +ĠR yan +Ġp ipe +Ġco ast +DE F +// ! +_ off +ex it +Ġult imately +imit ive +ĠKe ep +Ġhistor ical +Ġany way +ĠJack son +ock er +ER N +ĠU INT +y ntax +ER Y +is ms +Ġc n +Ġocc urs +Ġ; ; +Text View +A E +/ img +Ġy esterday +- default +Ġt iny +Ġpro c +Ġal ive +ĠRE G +. th +ear ing +.get Logger +< link +_ login +F older +ab c +lyph icon +н о +Ġnot iced +od igo +Ġed ition +im ator +. Enabled +.parse Int +Ġy ards +ĉĉĉĉĉĉĉĉ ĉĉĉĉ +Ġver bose +л Ñı +_B Y +.log in +.* ;Ċ +ĠM id +é es +Ġg lo +Ġbuild ings +Ġz e +ĠI ter +Ġt ube +ĠP ot +\ M +< th +br idge +ĠS cript +ĠM odule +Ġv acc +Ġinstall ation +v y +VisualStyle BackColor +ĠS M +.t otal +b at +Ġfind s +Ġat mos +Sub view +iz ard +Ġrepl acement +lic ated +ap is +Ġlog ged +ĠLe ft +G ui +_ Type +t m +P ad +Ġhouse hold +Ġre le +Ġpropos al +_CL ASS +:: :: +Ġinf rastructure +In ject +/ html +Ġad s +iz za +Ġm g +ctr ine +% Ċ +< html +- image +Ġatt orney +< m +(' , +Ġcan n +Ġprint ln +o ose +Ġy ellow +.ex p +p ayment +Ġtable View +aw ay +Ġopp osition +ĠAg ain +ĠH andle +Ġex clusive +in ar +é r +оР± +ĠC ODE +emp orary +Ġre act +pi pe +c z +. activity +Ġlarg ely +Ġdis s +ax y +es is +ĠR en +Ġc orn +.Use VisualStyleBackColor +d ays +Ġfr uit +In sert +_ enc +E st +_de c +ĠL uc +Ġü ber +param eters +P ERT +ex press +_pro file +Un known +Ġrev olution +.add ress +_re quire +Ġun iform +ĠP ack +l ar +ĠU ITableView +Ġdep ends +Valid ation +conf irm +O wner +Ġt rib +h et +ĠI de +ans as +L anguage +u et +ĠP o +ĠSte ve +Ġcont est +_DE FAULT +Ġapparent ly +RE EN +Ġfrequ ently +Ġtrad ition +ocol ate +S I +ĠArg ument +F ocus +ert e +ĠL ayout +Ġd x +Ġgener ator +ĠW ait +P olicy +l ights +.Ex ecute +P y +Ġbed room +ed a +ra id +ĉs ize +Ġan cient +Ġp ump +Ġd w +Ġ(! ( +Ġspec ify +( status +ĠF BI +.ex ception +Ġrem ark +ly mp +ant ee +Up load +ern et +é ¡ +in ent +ĠR ender +d m +ĠM emory +r ich +ĠT ools +Ġk ne +Ġper m +b ad +Ġd inner +.res et +Ġj Label +Fe ature +.S ervice +Ġ( {Ċ +Ġre ferred +.class List +Ġinit With +ĠText View +Ġne ither +Ġcount y +Ġ" { +ç § +Ġt ack +class Name +ĠUS ER +Ġre new +` ` +get Name +Ġb rown +Err ors +ert o +Ġsust ain +S O +let es +ĠIn valid +Ġen emies +un ge +Ġexist ence +err a +Ċ ĠĠĊ +utor ial +# a +p ay +char ge +ĠI re +ate st +Ġexp los +Ġf ired +N ER +ĠT y +ic ion +U ri +Ġobvious ly +ĠC olum +Ġ' + +ĠDe vice +- related +_ ARG +Ġv or +ĠLess er +_O P +Serial izer +Ġup grade +L ight +Ġc odes +++ ;čĊ +Ġwrit es +fo od +Ġé t +@ section +Ġtrack s +Ġserious ly +ch t +(size of +Ġimmedi ate +Ġscient ists +Ġ{ $ +_ ne +.Anchor Styles +Ġaccom mod +ĠHar ry +Ġs ight +ĠPale st +ersist ent +Ġ Ñĥ +- input +Ġco ordinates + · +W elcome +.con f +Ġgre w +Ġb old +ĠC PU +(m y +Ġperfect ly +Ġmom ents +ĠM ovie +- data +yst al +_W IDTH +ĠS creen +æ Ŀ +Ġdis ap +Ġredu ction +.Get Component +_M ODULE +Ġgener ic +Ġd y +all er +Ġc url +ĠB ody +Ġb anks +, t +av g +Ġev il +Ġmanufact urer +Ġrece iver +Column s +Ġing redients +ĉ out +qu es +.L oad +Ġslow ly +ĠT own +ĠC ell +_n ormal +_p refix +ĠAl ert +(" { +ä r +âĢľ The +ĠM D +Ġcour ses +ath an +é Ļ +oc c +ĠS ER +es ign +Add r += [' +(" ./ +] } +.f ont +ĠInst agram +ĠB order +od a +Ġh all +Ġr um +_b it +Ġs aving +_d own +R andom +_reg ister +( Context +Ġoppos ite +R oom +Y ES +ан и +Ġenjoy ed +_r un +C lear +âĢ ĺ +ĠF ord +on ic +ost en +"] ) +_ auth +// čĊ +Ġsuff icient +LE S +Ġph en +Ġo h +_c sv +Ġrout ine +.Are Equal +ay lor +Ġb asket +_COM M +rypt ed +S im +ĠSh op +Ġstud io +at os +( W +[ string +ä t +og a +Ġsh r +Ġs ick +An other +Ġdo ors +_N E +ĠTH REE +. order +raz il +Ġmap s +_TR UE +trans late +Ġnear by +Ġn ach +LO AT +b atch +Ġl ux +ash es +ang ers +âĢ¦ âĢ¦ +_E VENT +_ UP +Ġact s +in v +_M ETHOD +cc ion +Ġret ain +ut ch +ĠÐ ± +Ġknow ing +Ġrepresent ing +N OT +p ng +Con tract +Ġtr ick +ĠE dition +uplic ate +Ġcontrol led +c fg +j avascript +Ġmil k +Wh ite +Se quence +aw a +Ġdiscuss ed +ĠB ush +ĠY ES +.f actory +t ags +Ġt act +Ġs id +$ $ +ĠE num +Ġfr ames +} ); +Ġreg ul +'] ;čĊ +Reg ion +ff f +Ġc ro +( com +=" + +St udent +Ġdis appoint +RES ULT +Count er +Ġbut ter +ĠH a +ĠD igital +Ġb id +"> {{ +ing ers +ĠC ountry +_t pl +"] )Ċ +/ k +d ating +: # +ĠD ATA +yn chron +_b ody +olly wood +Ġval or +ip ient +o ft +UB L +doc s +Ġsyn chron +Ġform ed +ru ption +Ġlist a +Request Mapping +Ġvill age +Ġkn ock +oc s +" { +_fl ags +Ġtrans actions +Ġhab it +ĠJ e +ed en +Ġa ircraft +ir k +ĠA B +Ġfair ly +. inter +.A ct +Ġinstr ument +remove Class +.com mand +Ñ ī +ĉm em +( min +Ġo t +Ġcol le += s +time out +Ġid s +ĠM atch +ij n +z ero +Ġnetwork s +.g ov +Ġint el +Ġsection s +out ine +(c md +(d ir +ĠLI ABILITY +ĠB log +Ġbr idge +ĠC V +con vert +Ġ" )Ċ +ĠB ern +_P O +e val +( set +to ol +Ġpay ments +Beh aviour +Ġcon crete +Ġel ig +Ġacc eler +Ġh ole +_ o +TE GER +Ġgraph ics +O wn +Form atter +on der +Ġpack ages +/ a +ĠK now +Or Default +Ġdut y +W ait +н а +_rec ord +[ t +M esh +Ġon going +.be ans +Ġt an +Ġinter pret +ast ers +QU AL +Ġleg s +\ Request +- file +_m utex +ĠS aint +// # +Ġpro hib +( info +: = +lin ux +Ġb lo +ot ic +ĉf inal +_ex p +ĠSt op +ap ing +(s aved +_p ush +Ġe ase +_F R +pons ive +str cmp +: ĊĊĊĊ +ä» ¶ +ol i +Ġextrem e +Ġprof essor +Im ages +.IO Exception +Ġaddress es +plement ed +Ġincor por +Ġuse Effect +_O F +ĠD a +n ombre +IR ST +Ġdisc rim +Ġcomp ens +greg ate +anc ell +ach es +ĠC riteria +$ result +D estroy +Ġsecond ary +W atch +ĠS em +ĠMc C +Ġacad emic +U pper +:: ~ +ut ral +ĠD og +ad ed +Valid ator +Ġder ived +Ġset Timeout +ĠK en +Ġtyp ical +ĠB ob +Ġb ounds +ĠSe ason +Ġc razy +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +-r outer +itt est +ĠM ir +Ġemot ional +, v +c n +/ st +å ½ +on om +Ġdecl ared +> . +ail ing +Ġ/* <<< +Ġnorm ally +(M e +ev in +lik ely +Ġpoint ed +ĠSt ack +Ġw alls +. Vector +me an +] ]Ċ +Ġlist ening +ad v +Ġsw ap +IF T +Ø ª +. argv +ul s +< option +not ations +Ġemail s +ĠU kr +ast a +ĠTh us +ĠSt one +Ġappe al +. âĢĻ +Ġreg ulations +Pre ferences +ĠPh one +ul f +ĠD R +Ġtechn ologies +Ġpar agraph +Ġnecess arily +.e ach +< float +res a +Ġunder st +Ġf inger +press ed +-b y +if fer +w atch +ĠB a +A IM +Ġwe ights +ĠR on +') }} +[ self +-------- --Ċ +per iment +Ġto String +x ic +ĠC amera +! ĊĊĊĊ +aur ant +P refix +Ġinstit utions +: int +Ġex posure +p attern +ĠLin ux +.n umber +red ient +Argument Exception +ĠCh ief +" }, +Ġelect ronic +r ong +er d +sp Net +ra it +/ ', +ĠOh io +Cont rollers +Ġcontin uing +ĠT emplate +ĠE th +s z +/ env +En v +% . +art ers +) (( +ĠT ABLE +Ġà ® +per ature +pro gress +P res +ê ° +im plementation +Ġb ien +Ġstre ets +_M SG +New s +## # +: / +Ġcut ting +x B +ress ed +_EN ABLE +l ab +Ġca using +] ));Ċ +b ra +x FFFF +il ly +plet ion +w ill +_b ar +Ġstruct ures +ĠI mp +Û Į +Ġ< > +Ġ ---------------- +_B UFFER +.d ir +Ġpl ain +Ġpe er +g g +oint s +Ġsomew hat +Ġw et +Ġemploy ment +Ġtick ets +ir ms +Ġt uple +s is +$ sql +r ig +Ġcon version +Ġg es +Ġconfig ure +eg r +ĠC a +Ġ__ (' +ou ston +.t oken +Bl ack +Ġmag azine +A W +. IN +os ing +Ġbro ke +ĠC ru +DE LETE +Ġdestroy ed +(M ath +Ġappro val +-d om +ĠI II +table View +Ġdesign s +Ġcrush ing +Ġcons ent +dir name +om p +Ġc rypt +? ( +or ough +. o +ĉ list +ams ung +."" "Ċ +err ing +G oogle +_p air +_IN IT +rem arks +Ġg ear +F ill +l ife +} ")Ċ +Ġsuit able +Ġsurpr ised +_RE QUEST +Ġman ifest +att en +Ġfr ustr +ov ement +.c lick +Ġi i +Ġexp ansion +ig s +P arse +.Reg ular +R ob +_l ayout +ì ł +Ġtrans lation +ĠBe aut +B est +_C OLOR +< label +Ġliqu id +IT S +Ġpro d +Ġoper ate +UI Kit +Ġn atur +arg ument +_d etail +ĠCent re +Ġ" -- +Ġ}} " +lo cale +.t v +_se q +Ġup coming +Ch art +ĠDiv ision +Ġclin ical +Com pany +S epar +l as +ĠH un +: s +Ġhead ing +оР³ +Ġ" ");Ċ +[ id +b ia +Ġst retch +ic ide +Ġre produ +.pro ject +leg end +end ers +Ġrespons es +Ġon t +rit ical +Ġref uge +ĠL i +Ġ: ĊĊ +ĠTh ree +.cont roller +_IN DEX +_F OR +\Model s +j ax +ĉex it +Ġâ ĸ +Ġc overs +ĉ y +- . +IND OW +Ġfail s +in cludes +Ġf ault +Ġl y +ñ o +.s lice +ILE D +ĠP ur +ĠAs ian +_b atch +.M ax +v l +ĠCOPY RIGHT +Ġg iant +ĠMan ual +ĠC opy +Class Name +He alth +C ursor +IB Outlet +Ġt we +æ ³ +_label s +Ġcol lected +Ġfurn iture +Ġdeal ing +Control s +ĠHot el +ck s +Ġch ose +âĶ Ģ +od d +S R +Ù Ĭ +ì Ħ +Ġacc ord +ĠM ove +ĠM ode +ĠM ock +Ġthread s +++ ++ +ĠO ptions +Ref resh +ĠD id +'] -> +u cc +_ch annel +. abs +Ġ{ },Ċ +ĠW al +er ior +Ġmain ly +ĠDr iver +NotFound Exception +Ġcount s +e am +Ġ& = +Q uestion +ĠA li +Ġany more +d etail +t ail +Ġm ile +ĠF air +Ġs orry +Ġsurround ing +Ġad m +De v +Ġmari juana +ĠS ound +ĠA sh +F D +Te am +. port +Ġ[ ]ĊĊ +ub ble +Ġas c +Ġint ention +A cc +ch i +ust ers +Ġins pired +se g +CL U +Ġman ip +M etadata +Con nect +ĠB eh +Ġfind ings +Ġas sembly +w orld +Ġrem ained +Ġu id +( . +Ġm x +Lo op +ĊĊĊĊ Ċ +Ġfant astic +wh o +ak i +ĠB asic +ĠY et +ĠUs ers +ik ip +Ġhead s +ĠMich igan +_ it +ĠTor onto +Ġrec ording +Ġsub mitted +_var iable +medi ate +.graph ics +Ġst ood +Ġre ar +vel ocity +_M ESSAGE +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ro les +ĠT our +_ year +end ment +amp s +ĠIre land +m al +Ġyoung er +Ġstrugg le +Ġc able +ĠSD L +(' - +an es +ĠNe ed +.R ow +P ol +ĠP H +_s cript +ag em +ĠB as +_s pace +. loc +: i +ad r +Ġengine ering +it en +) & +Ġu k +ĠL ittle +_C OUNT +x A +Array List +æ į +Ġ" ")Ċ +An chor +Ġh ang +t witter +Ġcompet itive +.s rc +ãģ Ĺ +Ġtrans late +ĠCre ates +ook s +ĠR oll +'' 'Ċ +/ sh +s ome +Enc oding +.res olve +Ġdesign er +ĠSt orage +Ġz a +ĠN ever +Ġsomew here +Ġbox es +.s ource +Ġpy game +Ġgrow n +.t w +() ),Ċ +', [' +Ġoppon ent +(s rc +.l ayer +AP P +ĠAct iv +Ġguest s +ĠVAL UES +};ĊĊ Ċ +.n ative +Ġamount s +. RE +Ġcl one +Ġwer en +Ġ" << +_ ac +Ġbreak ing +Ġreli able +.P OST +ĠSk y +Ġ' & +Ġsaved InstanceState +ast ing +ill ion +com ments +ult y +.m enu +/ config +Ġ ĊĊĊ +T ODO +Ġpurch ased +_c or +ĉ auto +Compat Activity +com plete +_ graph +is odes +Ġsitu ations +ĠH or +Re ceive +âĢľ We +Ġent ities +.assert Equals +оРº +ĠS ans +v ince +rom pt += Ċ +Ġ/ . +.Se lect +yl v +Ġb att +A udio +Ġincreasing ly +.B undle +Ġexpl ains +the ast +. offset +Ġh al +Ġtechn ique +_l imit +Ġdraw n +AY ER +Ġfeature d +yy yy +at in +ph en +ach el +! \ +l ower +ĠG R +Ġp ag +ĠP arse +Ġt ou +ä¸ Ģ +D istance +Index Path +Ġh ell +s im +UT TON +Us age +elen ium +ĠF all +Ġ" .$ +ĠM u +Ġcr uc +Ġs ont +REF IX +Ġinter ior +ĠO lymp +.Auto Scale +par a +Axis Alignment +Ġr iver +D to +Ġwith draw +Re act +- class +b efore +_ alloc +Cont ents +ĠW as +I CT +Ġform ula +Ġindic ates +ĠĠĠĠ ĊĊ +_st ore +it ting +ĠIt alian +_S et +_re port +Ġp id +_V ER +Ġw ins +ĠCl oud +") {Ċ +ch ester +Ġden ied +Ġw ird +ĠSte p +Ġinvest ors +b old +_d isplay +ou ver +or er +Res et +Ġsurg ery +Ġstrateg ies +/m aterial +_ unit +Ġc ouncil +.P er +ĠâĢ ŀ +Ġre form +F ramework +Ġlist ing +_b tn +Ġb is +% d +eg as +Ġsudden ly +_S ER +Ġa o +_d irectory +f as +Ġprem ium +Ġtrack ing +ĠB L +Ġm ature +Ġbath room +Ġ'/ ' +ĠÄ ij +Per formed +Ġsold iers +arn ings +Ġwalk ed +- con +b ottom +Ġsurpr ising +Ġg ene +Us uario +.DE FAULT +ĠM IT +C ODE +ĠE gypt +p icker +ys ql +AT URE +d etails +ĠCon ference +In formation +ĠM ail +-d own +r aries +b ro +Ġsubject s +Ġ' * +è¯ · +or ient +: @ +ver bose +E F +Ġto ler +eng ers +Ġend point +Ġstr ange +Ġcol on +Ġpre ferred +de p +ĠE V +ARR AY +Ġw he +Ġp up +_n odes +Ġtalk ed +Ġinstit ution +db c +Ġex posed +te en +ĠFr ont +T T +_N ONE +\/ \/ +pro gram +Ġencour age +. ` +sh ire +ĠIsl am +e en +N I +' " +.W idth +Ġlik ed +Ġ{ ... +ĠSystem s +Ġvot re +Ġmanufact uring +Con verter +ĠIn f +ì ļ +D TO +Ġin ches +Ġ ठ+à ¹ +ĠChar les +B U +")) ;ĊĊ +ĠL abor +un n +Ġest im +m obile +ĠL earn +_C ALL +â Ħ +Ġind ices +Ġt ub +ikip edia +C ost +row able +ë ¡ +g age +Ġfunction ality +uzz le +em os +.l ib +Ġd ass +еРº +enn a +Ġsh ots +Ġrest ore +/ D +For Key +], [ +al ias +l int +.st ream +æ ł +_FORM AT +Ġsil ver +.re pository +Ġlegis l +.B order +_fe atures +Per mission +Ġhous es +ĠW ars +_COM P +Ġinj uries +Ġconstant ly +fl utter +EN U +ĠCon f +Ġrecogn ized +Ġpract ical +Ġde cent +B J +] ); +ast y +ĠAct ivity +-m ode +Ġsl ide +.IsNullOr Empty +ĠY OU +P ower +ind ices +Ġqual ified +Ġthrow n +h ello +ĠN ick +l ah +as sembly +ĠSm all +old ing +Sh ould +ĠSil ver +(saved InstanceState +Ġtog gle +.N ot +C trl +: nil +ĠCont inue +ĠB oot +æ ī +ĠM ur +d on +ĠF A +S napshot +Ġassoci ation +fo x +, a +az ione +] )čĊ +CT YPE +Ġf ade +ĠD ar +.n avigation +Ġl uck +SC RI +ĠDe ad +Ġterm inal +_LE NGTH +Ġeff iciency +Ġun w +Ġn arrow +iment o +( Color +ĠSe a +_ area +, A +_ opt +ĠHill ary +.t ask +ĠJ ac +ast ed +ĠAd am +ĠIl legal +Ġsearch ing +Instance Of +J ava +ĠForm at +Ġreal ized +ĠChild ren +Ġk il +(f rame +âĢĿ .ĊĊ +Ġscen ario +"] );Ċ +Ġincred ible +li x +IO Exception +ĠQ uest +il ty +Ġun lock +â Ĥ¬ +Ġre ferences +ĠV ert +B inding +eg ative +Ġwr ap +.d atabase +( content +B uf +ĠTr ad +ĠA ud +tr ace +.m ock +Ġther apy +ĉ L +.To Int +ĠKing dom +B us +ha ust +"" "ĊĊ +( end +.draw able +[ ];Ċ +ĠH ospital +Ġph arm +---- - +ĠA G +é d +> ");Ċ +Ġw allet +at able +) $ +Ġmonth ly +Ġdi agnostic +S ymbol +Ġiter ator +un finished +Ġimm igration +s r +RO W +(g ame +Ġclo thes +ĠU nt +Ġactiv ation +_C on +.h ash +Ġinitial ly +.H ash +Ġcut s +f ound +ĠSt ory +ÑĨ и +ac ao +_T YP +pro to +est r +-p age +ah r +Ġincor rect +ĠJose ph +TextBox Column +_st yle +ĠD aniel +s heet +Ġl iv +l ined +Ġr a +R untime +_ empty +sl ug +_ struct +ë Ĭ +m u +Ġper mitted +Ġreg ional +Ġsob re +ĠS uch +Ġ[ _ +Ġro of +.Al ignment +t imes +.m sg +Ġche st +ĠT ab +Ġest a +ä n +Ġsubs cription +( command +s pecial +Ġme al +") :Ċ +_ ctx +Ġclos ely +et ry +- be +ad el +ĠR am +ig est +ĠSpan ish +Ġcommit ment +Ġw ake +* >( +P HP +_ { +ck er +< List +_n ull +ĠRes erved +Ġin her +.Column s +.A spNet +_IN VALID +ĠParam eter +Ġex pr +} { +Cell Style +Ġval uable +Ġfun ny +In v +Ġst able +* t +Ġp ill +pl iers +ĠC SS +ĠCon dition +ĠS peed +ublish er +Ġoff ensive +ce st +ic as +Ġsp ark +ĠPro te +set up +IF Y +ĠT ax +Wh o +F amily +- for +. uk +Ġf asc +sv g +") ). +Ġbirth day +âĸ Ī +ve h +el led +Ġimport s +ĠIsl amic +T A +ĠSt an +we ather +Ġsus pect +e ature +enn es +W M +.m inecraft +av id +è ½ +.se curity +in os +G ood +Ġm arch +Ġposs ess +us uario +Con s +am ber +ched uler +Ġhor se +ç ½ +(b ody +ĠTrans form +_de code +.s vg +Ġf oo +Ġd ella +ext ends +am er +Ġprocess ed +ĠH arr +ĠA I +Ġk o +CH AR +( % +Ġt ap +({ ' +c roll +D OM +Ġte a +Ġre in +Ġworld wide +_f n +sh a +Ġb ir +ç ões +="# "> +Ġrepresent ed +ill er +(ex pected +Ġd ance +Ġvisit ors +.con cat +-b it +UR RE +ĠR og +v p +ip h +ĠL LC +it led +iam i +C oll +_re al +_sh ow +_f older +Ġd ar +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġl atter +arch y +Ġb ow +Ġout come +ĠPost ed +Ġris ks +ĠThere fore +Ġowners hip +Ġpar allel +Ġp ending +ge ometry +Ġrecogn ize +ST EM +ĠC P +Ġimm igr +IT LE +ĠĠĠĠ ĉĉ +conn ected +Ġsm ile +(d ocument +\ Component +vert ical +Ġconsum ption +Ġsh oes +. impl +un ks +. ";Ċ +Ġfood s +_ );Ċ +.assert True +Ġp ipeline +Ġcollection s +Ġearn ed +ĠC ert +Ġpartners hip +( action +Ġc d +ĠV ery +Option al +Ġscre ens +Ġtit les +ener ator +Ġab andon +k ind +IL TER +Ġclos ing +lic a +_ inter +Ġcamp us +set ting +S prite +ãģ ¯ +_re ply +To List +: \/\/ +ed e +Ġfol ks +Ġbo at +( argv +Ġperman ent +Ġcarry ing +Ġconserv ative +import ant +. img +ĠIm m +Ġdim ensions +al and +s ingle +Ex it +-------- -- +ari ant +tern al +Se conds +ĠIt aly +ot lin +.Res ume +=' " +) == +cept or +Ġs ca +/m ain +Sec urity +_d at +Ġlet s +Ġa qu +Ġwhen ever +b erry +Ġact ing +ant i +p d +& gt +æ Ń +Z one +T oday +! . +To Props +ab is +it able +Ġg al +] { +iz ona +Ġin contri +N ET +/// Ċ +[ in +_s ave +Ġex em +ĠK enn +Ġev olution +var s +_st ats +- only +ĠColor ado +Ġwatch ed +b our +Ġsever e +Ġprofession als +port ion +Ġguar ante +Ð ³ +Ġpush ed +ĠG i +ï ½ +Ġt um +ĠA z +ĠEdge Insets +")) ;čĊ +is se +. ac +Set ting +Ġapprec iate +ĠValue Error +Ġsur ve +ĠR ole +. Inter +plot lib +j et +d am +Ġplatform s +te le +UT O +ĠInt ernal ++ : +} ;čĊ +Gener al +\ Entity +Ġlawy er +qu iv +ĠPost s +is o +Ġacc um +ob e +Ġmark s +Ġ] ;ĊĊ +ĉ text +.s uccess +cur r +as a +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +Ġth in +_ over +are st +ĠO s +( address +Ġvel ocity +Ġ[] ;ĊĊ +=" ../../ +ĠPr iv +b ow +Ġguar antee +% ĊĊ +Ġeval uate +.LE NGTH +Ġin ventory +q a +_de bug +.On ClickListener +Ġl ies +Ġassess ment +dat etime +.background Color +Ġ*/ čĊčĊ +ra f +un wrap +ĠF oot +Ġnot ify +Ġlow est +DO CTYPE +Ġl anguages +ex tra +- back +Ġein en +tem plates +_p ass +ĠM ust +Ġest á +_c ore +ĠSc ot +A I +Ġb ias +ations hip +Con stant +Ġprogram ming +In s +uspend Layout +ĠPRO VID +ant es +Ġsh irt +in ated +. OK +[ a +Ġthink s +? ĊĊĊĊ +Ġregard less +ĠMag ic +ul ating +ĉ class +add Group +RE ATE +ĠS U +Ġsim pl +c opyright +Ġb unch +Ġun iverse +ĠE rr +Ġpresent ation +c ategories +Ġatt ach +.s ign +_A C +Ġdisc ipl +Ġregular ly +Ġprim arily +ink s +[ [ +.r and +.sh ould +ownt own +=" ' +Ġs ans +Ġsupport ers +se quence +G O +. .ĊĊ +ĠS pr +Ġcare fully +U IColor +dest roy +Ġtod os +ĠOR DER +ott ed +Ġd ont +aud i +_ player +g re +ĠO il +< body +_st ack +.P adding +ĠProduct s +Ġpriv ile +Ġinj ured +ĠF urther +Ġal ias +.Resume Layout +_LE N +Ġs es +'] ;ĊĊ +cre ens +Ġdirect ed +.S uspendLayout +od ge +.A t +mark s +ĠUn ivers +ert s +ĠE sc +Ġnav bar +Ġutil ity +agnost ics +Ġin ject +ĠD NA +Ġ" ," +am ar +Ġe u +Ġrestaur ants +_p ut +ut ers +Tool Strip +t w +ist ro +Ġz oom +Ġleg it +pec ific +ĠC ome +Ġlocal Storage +Ġabs or +.P anel +ĠDesign er +Ġo w +IC AL +_ uri +(f ield +Ġsup erv +Ex ists +Ġrespect ively +ĠSt and +Con f +uss ian +Ġar c +Ġ nd +uck s +Ġre str +Ġseason s +ĠCh apter +ĠSw itch +p ic +Ġh i +load ed +Ġfl uid +-b tn +Ġrun time +. it +B N +Op acity +as ant +ry ption +-n ative +Ġta ught +å ¯ +ag ment +Ġm ul +Reg istry +_ grid +ĠBro ok +: Set +Ġm ongoose +AM ES +inner HTML +Ġs oci +ĠInt el +get Id +C md +Ġaccess ible +r ames +le ton +Ġ__ ( +ĉ delete +ĠS quare +" ĊĊĊ +Ġbu cket +avor ite +ĠB reak +++ ] +Ġbr ush +Ġt ensor +/ http +T ile +Ġfunction al +Ġ" * +wh el +Ġt ent +ĠChar acter +Ġse es +. ST +B ig +Ġext ern +Url s +)) )), +ĠJ r +.B uilder +. ; +n l +_ Init +ĠH ER +ż e +mys qli +_ icon +v an +Ġfeel ings +Ġle an +Ġhop ing +T V +="čĊ +b est +all as +ent ed +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĊ +_con nection +Ġrep o +en abled +аРº +Ġsh a +Ġmembers hip +Status Code +in ating +_s m +_c ustom +_ weight +Ġc ss +St at +_ env +link s +TR L +ĠH it +, r +up id +Ġop ens +Ġg ent +_v is +Ġj oy +< w +_c ost +ĠPy Object +ren ce +ĠGeorg ia +ĠBro ad +m ma +â Ĥ +p f +Ġ" \" +Ġ( & +om o +Ġliter ally +Ī ĺ +met ric +Ġb ars +z ed +(w indow +ĠIsrael i +Ġform al +ident ifier +.d ao +ĠDe ath +% ;Ċ +Ġdecl are +ar ms +RE AM +PERT Y +Ġconsequ ences +to ols +Pe ople +ĠWh ich +> ();čĊ +.de code +_A CT +Button s +.f loat +.F irst +ë ¥ +ĠPol it +ĠX CT +T ags +ĠCG Float += str +Ġle af +- check +ĠI ss +.s ystem +log out +ach t +Ang le +s in +ch art +INT ER +ĠN UM +B asic +.P roperties +ä¸ Ń +_ change +ĠB razil +Ab stract +Ġ: +: +_ use +а л +ĠL y +IB UT +Ġout er +Ġ-- >čĊ +Ġrel ief +l ap +qu er +_p arent +he ap +LO SE +Ġcomb ine +ĠR ose +ow ers +Ġproced ures +ĠS ort +an im +var iant +eh icle +Ġsign ing +Pr imary +c urrency +Ġsex e +o en +th eta +em an +Ġimpress ive +(' _ +ĉ U +ĠText Style +_c nt +Ġs lice +(' : +Ġunderst ood +H is +Ġinform ed +Ġn ick +(T AG +h d +Ġelection s +est ure +ĠS anta +ĠCo ast +.p df +inc iple +.cl one +b orn +ut a +Ġl icensed +C r +Ġb read +ĠH ouston +Ġn od +Ġhop es +ĠCG Rect +Ġgu ilty +.g if +Ġro se +.Com mon +T ip +AN K +ĠF C +D uring +ĠSym fony +Ġdef ensive +k m +) > +arch ive +ĠU RI +ycl ing +- o +ĠWe bsite +AM P +ish ment +Ġdo ctors +D irect +AR I +ĠRed irect +ier en +_d ist +y o +ĠPro gress +Ġz um +Ġmem or +ĠE D +Ġj ur +æį ® +_T ABLE +Ġu uid +Ex pr +. head +(' % +point er +Ġest imate +ĠG reg +Ġlo ader +Ġi OS +Ġm ens +[ y +Ġref used +Ġprec ision +is ch +ĠA CTION +Cl oud +s With +( ret +_ADD R +_con f +(d f +Ġlock ed +Ġr ising +ãĥ» ãĥ» +ĠM s +Ġscen es +_EX T +_ raw +_ the +pe ople +Ġre con +ĠF un +Ġb less +ĠUp dated +ü n +ĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +pe ction +Re lease +.log ger +ĠS Y +Ġcoun sel +ur d +_ true +Ġevery body +iv ot +Ġh ence +ĠN AS +Ġoppos ed +unk nown +ĠDES C +ĠCh air +fa iled +ĠIN CLUDING +Ġwrit ers +{ }Ċ +ÃŃ t +_c opy +} : +ĠB at +Ġconvert ed +ed ing +pl acement +ĠH ost +S ound +и м +Ġs ought +m id +Ġsal ary +og g +âĦ ¢ +b ul +Ġw ir +valid ator +_ST AT +.st ore +ĠB attle +ı n +Ġ-- >ĊĊ +Tr ump +d ot +ĠCON T +.f etch +Ġcontin u +w as +Ġfra ud +_t mp +mit ter +.p ictureBox +G A +Ġt ournament +. Input +[ r +ex ion +cent age +ĠKore an +und ef +ĠAv ailable +resh ape +Ġk it +ĠStr uct +ĠS UB +An swer +_l ib +.t witter +Ġo re +ĠDr agon +.Ex t +, k +Ġexplan ation +ref s +ĠDr ive +ĠTr aining +.H as +int age +b ig +olog ist +enn is +Ù ĩ +Ġch icken +ĠĠĠĠĠĠĠĠĠĠ Ċ +ç Ľ +ãģ § +Ġpe ak +Ġdrink ing +Ġen code +ĠNE W +m alloc +ĉf printf +Ġ= ================================================================ +in cluding +Ġprincip les +ĠM ah +st orage +- key +Ġkey word +% ; +Ġtr ained +.con trib +Ġk v +__ ':Ċ +ĠB oy +param eter +Ġsu ite +Ġthous and +Ġco ordinate +-g enerated +íķ ĺ +gener ated +Ġad mitted +Ġp ussy +# w +Ġsw im +un ion +N a +ĠRoy al +.ch annel +Up dated +_RO OT +Ġv ital +ra ction +ĠCrush er +Ġpre ced +Ġhor izontal +Blue print +Ġattr s +Ġsm oke +Ð Ĵ +. Equals +F B +ĠRes ources +roll ing +Ġpass es +ĠN um +rot ate +et ype +\ ", +Ġsens itive +Ġt all +? âĢĿĊĊ +Pro xy +i y +_ section +âĢĶâĢĶ âĢĶâĢĶ +br id +Ġcirc uit +at an +EN C +Ġdr iven +Ġvot ed +Ġeduc ational +Ġinter action +abet es +Ġt one +ĠInitialize Component +Ġmer ely +Ġì ŀ +co okie +_ div +ĠUIL abel +vel y +} );čĊ +_ ENT +#+ #+ +art icles +ĠSou thern +Ġstrong er +ĠG iven +ĠE ric +ĠI R +ab stract +U nder +n able +Ġincre ment +ov en +Ġco in +_t imer +Ġsuffer ed +ĠF REE +'] ." +ĠQue en +st ats +Ġmeet ings +Ġenter ing +Ġalong side +(s ession +it als +Ġfound ation +ĠC redit +. div +_ ALL +pc ion +_st at +ick ing +Default s +_s rc +Ġoutput s +/ B +Ġent hus +-b l +.Fore Color +ĉ temp +F ace +Ġinter act +Ġwe ird +M ount +re ll +ud ents +Ġrequire ment +ĠS us +I ER +Ġe lected +re ference +ĠM E +Ġserv ers +.w ait +Ġsnap shot +il ton +Ġtri es +Ġt ipo +.T ime +> w +Ġmount ain +Ġp ounds +Ġ[ ... +ex ists +Ġng On +_M AP +Ġf lying +xi ety +ĉ value +_D B +un o +Ġse ats +T URN +. author +! ) +or ce +Ġindic ated +.s in +Ġass ignment +im iento +ĠF rame +_g en +in ery +_ ) +m essages +.set tings +ĠMe an +ĠM useum +ir q +att ach +ĠPalest in +_ QU +_t ags +Ġcas ual +em en +ASS WORD +$ s +ĠC irc +оР¹ +et ric +/ P +Ġep och +< head +_C MD +Ġg it +Ġpen alty +or ph +_ users +ours es +.Date Time +atern ion +_pro ject +Ġsuper ior +ĠD am +ĠSe attle +X Y +> The +ĠA k +Ġgr ass +/* čĊ +(d is +Ġgun s +Ġt b +ĠK evin +. args +ĠA h +op ed +( J +column s +arg uments +ĠWith Events +_f ull +ĠDef ense +S imple +Ġdeath s +Ġext ensive +ĠSt ill +ĠEx pression +ĠAg ency +Ġperform ing +F X +Ġus uario +U AL +S ide +od os +apt op +Ġcred entials +_c ap +at ient +ĠDis ney +Ġa i +Ġch ip +Ġvol t +.make Text +%%%%%%%% %%%%%%%% +Ġbelie f +_LO C +ĠC ivil +N avigation +Ġreve al +Ġviol ent +ĠF il +Ġc atalog +em ed +sc an +. control +Ġconstit ution +C ountry +Separ ator +_A PP +top ic +uet ooth +M IN +Ġdes criptor +y t +ET HER +Ġdistrib ute +' }Ċ +.tr im +.L ine +Ġl bl +assert Equals +ĠD et +omb ok +( width +Ġt ort +ĠEXP RESS +ac o +Us ing +ĠBr and +w all +EM ENT +ĠComm unic +< uint +ĠG UI +EG IN +ĠR ange +/ i +ĠT aylor +c ost +Ġrespond ed +ĠTh eme +n ce +IS H +Ġfeat uring +Return s +ĠK r +Ġ .Ċ +Ġn am +_c b +Test ing +Ġ{ }, +y al +.f ield +Ġ/ = +_SH ORT +m ates +Test Case +ain less +Ġeval uation +_ ITEM +ĠPac ific +ĉ k +Ġc ant +ĠR os +) s +Ġf et +STR ING +ĠDis pose +g al +ĠJ oin +ĠP orn +ĠCath olic +AR GET +cp u +ç łģ +.sc roll +IS ING +ifest yle +anc ement +Ġm erc +ĠB rowser +eter min +Ġover flow +Av ailable +Ġbott le +: UI +ific ial +Ġco ord +clar ation +Ġcon j +G LOBAL +ok u +Ġk wargs +cond itions +ul um +Ġg enu +ĠH ero +å İ +Ġun expected +ĠDAM AGES +Ġk a +ĠC ould +UP PORT +ĠPh otos +Ġconf ident +Ġdet ected +de g +rg b +Ġstrong ly +Ġ} ;čĊ +Ġ) : +Ġle ct +urs ive +RO L +ĠWe ight +Ġent ertainment +Ġ) );Ċ +Ġg onna +Ġb b +.d o +G S +Ġmist ake +D L +ĠPROVID ED +ear ning +L imit +iss ions +[ v +ä¸ į +ir ty +D el +Ġunder lying +pre ne +Ġj aw +ĠD I +pe er +Ġobject ive +Ġde posit +Ġk on +Ġes p +.set Visibility +/ login +< typename +Ġfr anch +/ e +Par allel +Ġsc ored +ĠH on +ĠV ill +ig a +Ġant icip +_ assert +ĠO pt +Ġdescri bes +w an +m ount +Ġmonitor ing +Ġt out +ëĬ Ķ +}, { +................ ................ += int +Ġc ust +---- -- +Ġatmos phere +P AR +ort e +IS IBLE +ĠI ron +ĠNot ification +.log ging +ĠBO OL +-p oint +Ġaf raid +ent a +Ġtom orrow +@ implementation +Ġeng age +ĠAn th +ĠF loor +ĠU l +To ols +Ġb ab +Ġcare ful +ãģ Ħ +Ġcruc ial +Ġcalcul ated +ĠS A +Ġw y +D X +_T AG +ind ed +Ġj et +ĠEngine ering +.M AX +en z +v d +Ġpublic ation +Ġ## # +Ġfac ed +ra ham +ĠC apt +As set +ĠCon stants +Ġlo ans +_ IP +ĠF ish +Red uc +_m at +Date Format +_m e +[] [] +Ġintegr ity +ĠC ourse +lob als +Ġfac ilit +Ġem br +ĠN g +.S ystem +Ġmanufact urers +Ġpro ven +.on Create +Ġal arm +Ġ § +Ġcomm only +ic os +æĸ ° +ĠSt ation +} ). +ĠF ilm +w i +ç ī +Ġeng aged +St ats +Ġgovern ments +Ġafford able +_p roperty +Ġag es +(' -- +Ġf ör +ĠProf essor +Ġhy dro +P ush +Ġorgan ized +Ac cept +é m +_c ell +Ġn b +p b +Art icle +Ġrem oval +Ġauth entication +ĠF R +l ide +Ġple asure +ap ol +Ġpart ition +ĠS ide +Ġcr imes +Ġdem o +hold ers +ĠPak istan +In struction +Ġexpect ations +.sc ene +Ġ' ) +h es +ino is +_P ro +Ġm olec +and al +_sh ort +Ġdefault s +Ġn ations +in en +Ġr t +O CK +P acket +S B +ĠSH ALL +_cont ents +ise conds +vert y +á t +G uid +n om +Ġcon clusion +. Update +Ġlo vely +Ġem it +b ec +ĉĉĉĉ Ġ +Ġintel lect +Ġb rew +ec ycle +F ire +Ġad mit +Ġar bit +Ġarr ang +ĠM IN +M ail +ĠN ative +C ur +Ġcon vent +.R untime +" }Ċ +.R un +Ġprint ed +Ġconven ient +. ar +m ock +ĠAdmin istration +ãģ ¾ +Ġelect ron +fl ate +Ġl ombok +Ġjava fx +n h +Ġsup plies +Ġvisit ing +ah l +Ġpow der +Ġult imate +Ġorient ation +ut as +_s cale +Con firm +ph ones +ĠOper ation +/ T +_IN TER +Ġair port +Ġmet rics +Ġphen omen +a udio +Ġm ai +( K +h u +all ing +rodu ction +ĠTrans port +ĠNOT E +æĸ ĩ +Ġfew er +_T IM +ì § +к и +A ge +F IN +Ġì Ŀ +ĠAt tribute +group s +er k +at to +. define +.AspNet Core +ategor ia +ĠS ir +( form +< User +. round +_d ay +.A ll +Servlet Response +.N o +l arge +IG H +qu ent +Ġvir us +Ġret ro +Ġim per +Bit map +Ġv ice +Ġoff ense +ist e +ĠA UTH +Ġê ° +ToolStrip MenuItem +G u +Ġr ape +ĠDav is +Ġover whel +: flutter +- table +ĠCon structor +Pr ivate +e ven +ch r +Ġap plies +_at tribute +Ġcon tribute +E VER +L ines +ĠAf ghan +Vis itor +ĠS L +se ason +C U +Ġintrodu ction +Ġmat plotlib +Å ij +Ġnewsp aper +âĢĶ and +< tag +Ġin i +Ġd iverse +Ignore Case +ĠU r +Ag ent +Ġb ull +.em it +( Exception +ar Layout +Ġincred ibly +ĠTr ust +={ ( +- nav +Ġe quals +Ġl ady +ĠP od +d isc +al am +ĠI V +â Ļ +iv idual +ph i +add ed +Ġdifficult y +Ġcomp act +ĠAction Result +c ers +_class es +Non Null +Ġqu it +Ġp ou +S witch +ir s +- test +ĠK ind +ĠCal endar +Ġstream ing +} ', +S W +Ġst ead +oc a +Ġprov ince +Ġcol span +Ġperson nel +ĠE mployee +Ġprodu cer +Ġevery where +od b +Ð Ł +bs olute +act ivate +Ġgr inding +ĠBuild ing +ĠSand ers +(s c +ĠOff set +//////// //// +} ;čĊčĊ +({ " +Ġscan f +ĠY Y +ĉdef er +Ġj ew +Ġrestrict ions +.m p +[ l +ä¸ ĭ +label s +red icate +aw esome +Ġw aves +Ġcon front +Ġmeas ured +Ġdat as +_ex it +ot ton +Ġshould er +ask a ++ # +ĠĠĠĠĠĠĠĠĊ ĠĠĠĠĠĠĠĠĊ +Ġtro ops +ĠU nd +_c ard +w ich +Ġn ous +Ġ"/ " +s b +Ġcommunic ations +Ex port +Ġdec ode +th s +inter pret +By Name +ĠSp irit +ed ges +O LE +ĠE M +t it +ĠTh rough +Ġb io +ĠP ackage +or ne +Ġ} . +` ;Ċ +Ġok ay +ĠZe aland +ident ity +(n ext +ĠB ang +Lib rary +Ġheav ily +il on +Ġdi pl +Ġrot ate +put s +) ',Ċ +ĠData Table +Ġmay or +.to LowerCase +Ġsome how +ĠNor thern +al c +Ġcap abilities +Ġv ibr ++ Ċ +ĠS u +ĠRes et +_m ean +Ġc ig +.cl oud +ĠB and +ĠF actory +ĠAr izona +_ io +op her +Ġconsc ious +Ġà ¶ +\ Controllers +_s peed +ĠF ac +_C om +ĠB ible +w en +ED IT +Ġun n +ĠSt aff +ĠIn n +Ġmechan ism +ĠM embers +Ġmigration Builder +'] .' +.get Int +< void +ĉf ree +oid s +\ Support +Ġautom atic +Ġch ances +Ð ¶ +Ġcomp licated +[ row +ah oo +Ġ}ĊĊ ĊĊ +Model s +W in +Ġt ape +ir us +iz on +on omy +(" _ +: . +.st ereotype +( env +_re ct +(w ith +Ġassert That +Ġcon straints +put y +E mployee +T D +Ġgu itar +ĠJew s +.pro cess +Ġf iction +ĠSh ared +âĶĢ âĶĢ +Ġprop ag +.N et +Ġachie ved +ĉ Q +Ġn urs +Sh ared +_FAIL URE +Ġbeh aviour +Ġcol s +ism o +Ġfem in +Ġchalleng ing +Ġpost ing +enc il +Ġcapt ured +ĠD ou +( word +ĠTur key +pan ies +Ġre putation +ORM AL +Ġelig ible +prot ocol +id as +(f rom +Ġfin ance +- per +Ġg otten +H A +d uration +ĠP arent +Ġin vent +Ġre start +ол ÑĮ +r ition +(r s +< bool +i ert +Ġmod ification +ĠT X +readcr umb +b ank +$ / +ĠMill er +] ),Ċ +.Check ed +Ġsac r +se curity +Ġp ose +ĠBr ad +Ġfit ness +Ġannounc ement +ation Token +Ġserv es +ne ed +Ġge ometry +AR S +æ Ģ +andid ate +Ġs prite +_s plit +We ek +ad ies +> (Ċ +?> " +Ġ/// Ċ +Ġein er +Ġweek ly +ĉlog ger +_p op +_m an +Ġmigr ations +Ġask s +Ġb s +Ġfall s +.W here +- height +_fe ature +.M in +Ġhy per +Ġvol atile +Ġtw enty +Typ ography +Un able +D et +, f +-m od +Ġsett lement +Ġcontract s +n ome +B ad +ĠB rian +(user name +!! !! +Ġh ack +.F ield +H R +ĠJ ordan +iz a +Ġ ł +ĠSh er +. header +( other +ĠD ub +( op +ĠR ound +Ġv ie +Ġap pl +ĉ J +ĠIn sert +ĠL P +reg on +ĠM PI +Ġan chor +ac a +ø r +Ġa de +anch or +que e +ĠTree Node +Ġtarget ed +Ġla id +AB EL +v et +ĠOr igin +A nt +. ');Ċ +ex pect +ed Reader +ĠM ajor +Ġin ch +Com par +Ġpre view +Ġill ness +ĠCONTR ACT +ĠInd epend +u uid +Ġn ome +Ġt c +ĠA venue +is an +Ġph rase +_m ove +") [ +Ġprov ision +Ġconcent r +_ IR +ĠU t +() + +Ġn as +! , +ĠRob in +i ations +at itude +Ġp x +ĠWith out +/b ash +ek t +re ement +Ob server +ĠReg ion +UBL IC +Ġ{ // +K N +å · +Game Object +å ¾ +enc oding +Ġ** * +project s +Ġt k +Ġche ese +EM PL +ar o +Ġا ÙĦ +Ġcons ists +ref resh +ure au +ĠSc anner +Ġso il +Ġfl avor +Data Source +Ex ecute +ени е +Ġsh it +åĪ Ĩ +< any +Ġretrie ve +Ġbelong s +.st rip +abs olute +Ġexp anded +bo y +): - +Ġresc ue +.J Label +Ġre ly +Ġal ignment +-f amily +Ġre nd +OLUM N +Ġb orrow +Ġqu otes +ĠL ew +Ġsh ower +ĠDE LETE +_lo op +! "ĊĊ +ĉ re +Ġattempt ed +aver age +ĠP aint +quis ition +ol en +Ġliter ature +ĠRe ference +_TEXT URE +ĠS eg +ĠInd ust +ct ype +D UCT +_H OST +ĠTr ade +Ġpl ugins +Ġbre ast +ul se +Ġcreat ure +ãģ Ļ +ĠW i +Ġsup plied +c oll +! (" +Ġfuck ing +ĠCh rome +ĠU ri +ĠN ation +Ġvert ices +T HE +ĠOr iginal +on de +Ġsh arp +Ġcook ing +Ġ{ /* +ĠPs ych +ĠH ollywood +=$ _ +.D ock +Ġg er +Ġb one +_con n +_se c +ys ics +Ġ= " +S al +s f +Ġdeep ly +ang les +T erm +b ell +ĠQu ick +ener ation +adio Button +åħ ¥ +}čĊčĊ čĊ +Ġcapt ion +l c +ĠE L +, [ +ĠĠĠĠĠĠ čĊ +ret t +(m ethod +ĠFl ash +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +W ISE +.s cale +Ġrough ly +_ child +m emory +ay ing +Ġinitial ized +in ator +а ÑĢ +Ġsc alar +ĠH o +ai res +(c olumn +.de stroy +P ACK +Ġh em +ang el +_S UB +. qu +Ġ × +DE FAULT +pos itories +ĠL ength +ĠF ast +Ġsign als +Ġ// $ +ri ers +Ġd ummy +AN Y +Ġperson ality +Ġa gricult +Pl atform +ER O +ĠT ra +Ġen orm +ĉ W +Action Result +Ġa ver +[ str +Ġ' -- +.S printf +Ġdeb ut +Ġ Ñĩ +h ex +_ utils +Ġp b +U ITableView +Ġz ur +. encode +Ġv ag +.error s +о н +Ġm r +ĠA ward +Ġc pu +Ġpress ed +' est +ĠF estival +' T +Ġa k +res olve +.m e +Ġn ic +Ġgen re +Ġat trib +ĠMo on +Ġarr ive +ĠD ating +Ġt m +.Config uration +. red +Ġgl m +Ġst ations +sw itch +Ġt ied +äº º +Ġ/ >Ċ +Ġsubsequ ent +pos able +-fl uid +Ġth orough +Ġpublic ly +apt ers +ĠWil son +_P RE +y ard +ä ¼ +ĉ in +Ġre vers +Ġbul let +cri bed +nes ota +Ġ($ _ +ann on +c ursor +Ġclo thing +ĠM ulti +: ', +Ġv ess +ordin ator +Ġein em +C annot +Ġar med +ĉ V +ä¸ Ĭ +.F lat +ĠS ep +ĠSub ject +_f ont +Ġcharacter istics +D one +el n +######## #### +PO S +Ġd ensity +ĠPl atform +- items +Ġo vers +Ġpush ing +ç ¤ +.Con nection +_ term +Ġinitial ization +________________ ________________ +ç ¬ +.d ocument +les h +ĉd ocument +ĠP in +ç a +Ġdefinition s +.P ath +_W RITE +Ġ ĉĊ +? >ĊĊ +Ġter rible +be an +ick ets +ĠS V +B uy +(t ask +Ġreg ime +g oogle +Ġcr ack +.vis it +N UM +ener gy +Ġstr uck +_s ample +.p ayload +Ġre vis +ĠSc ene +Ġp g +Ġbreak fast +URRE NT +.char At +_ex ception +ĠAnt on +Ġguid elines +Ġex haust +ĠFin ancial +Ġind ent +Ġdes ktop +H idden +F ailure +Ġpr inciple +Ġ iv +Ġse ks +n etwork +Ġnumber Of +ĠAl bert +ĉ long +, . +Ġz eros +f ade +ĠT yp +ĠT erm +ĠAr ts +.App lication +Ġbeh alf +æĪ · +Ġm ere +(` ${ +Ġaware ness +elp ers +f lix +Ġwe igh +Ġestim ates +. child +/ O +ĠBit map +.b ottom +Ġ************************************************************************ ** +Ex pect +ent o +ĠFor um +ver al +Ġj ail +Ġab ilities +ĠH OLD +ĠC it +Ġd ynam +Ġgr ay +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉ +.next Int +ant ly +ĠAR ISING +( private +Ġreject ed +ĠN ic +Ġle ather += {Ċ +aly tics +th etic +.T op +.P age +={ ` +Ġ ;čĊ +de pth +m ann +W D +ĠS om +.R ight +Ġ) }Ċ +Ġtr ait +Ã Ĺ +i ac +Ġr v +S ample +.X ml +opp ed +ĠÑ Ħ +list s +Ġt ear +ivers ary +.c ollection +ĠCon stitution +ĠHttp Response +Ġbr ill +ĠP rom +h over +ĠM iami +Ġarg ue +_f loat +Ġ ãĤ +Ġn at +ĠT al +Ġinteg ration +(c ur +Ġrem oving +Ġco eff +ĠTh ough +Ġfore cast +ĠV egas +S ite +Ġtr ab +ĠHen ry +- i +Ġinvol ves +B T +Ġs lo +In voke +Ġl ucky +r at +Ġ? Ċ +Ġhand led +(f d +cont ents +ĠO FF +R F +Ġst y +ĠM otor +ter y +t ax +M AP +ĠMr s +Ġph ones +ĠUI View +")) );Ċ +( dev +ĠIr ish +Ġw s +D I +_OFF SET +ĠEvent s +Ġst ages +Ġ} // +Ġhab en +ST ANCE +ĠS in +ĠM oney +(t op +Ġappoint ment +VER SION +met adata +_com ment +Ġcolle agues +map s +â ĺ +Ċ ĉĊ +( al +_re q +Ġf ut +Ġarchitect ure +ĠWH ETHER +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +_s creen +Ġstyle Urls +Ġmon ster +. up +ph ia +Ġprocess or +ĠT err += ', +ĠMan ufact +ĠN T +k el +ib ern +ĉf ile +A li +rient ation +Ġ// ! +ap ore +ane ous +ĠC reat +f older +Ġh ay +Sup press +( left +Ġe uro +Ġdis claimer +ustr y +sh ips +_f d +ĠF a +_in sert +Ġro l +if ting +ĠCom ments +_b r +Ġloss es +ĠAdd ed +ch arg +Ġп о +_s ystem +ĠS ometimes +ĠSp ain +(g roup +ial is +Ġdoll ar +ĠAr gs +qu ires +ĠT en +.s css +Ġsurv ive +us age +Ġj un +im iter +ï¼ģ ĊĊ +Ġfif th +t oggle +Ġdecl ine +($ " +(L ong +ing e +Ġpil ot +-l ight +-r adius +Ġpod cast +Ġnatur ally +P ages +ä¸ º +ĠDes pite +Ġlight ing +Ġcr ate +ĠB inary +Ġredu cing +Ġe leg +ĠM ouse +ĠTest Bed +Ġbefore Each +_ ARRAY +Red irect +Ġf lood +Ġsh ips +Ġelectric ity +)* ( +ê ¸ +ĠV iet +her o +Ġd ia +ĠK ent +he art +Ġthreat s +_ acc +Ġs ymbols +is chen +_in st +C riterion +ĠT IM +. Height +Ġ âĢĻ +();ĊĊ Ċ +Product s +_S P +ĠC y +Ġdepend ent +est e +Ġdat os +d it +аР² +IGN AL +Ġless on +"> ' +ĠC over +ĠH ope +ĠT imer +Ġd ad +vid ers +ĠPh ot +/ ? +rop y +om ing +as ion +Ġ\ ( +ĠE T +ĠRe ading +Ġep isodes +l m +ech a +Ġne uro +Ġhar mon +Ġlib eral +- ind +D ATA +Ġevery day +Ġdiv ided +ĠActive Record +fig ure +U A +ä ¹ +riend ly +te ch +.game Object +иÑĤ ÑĮ +Ġmo on +ft ime +Ġno ch +ĠT ORT +ĠV M +.in itial +( child +Ġmus ical +Ġo c +b as +ĠH ay +_l ong +Ġmem set +ile y +adel phia +S V +ro at +_t x +Ġl on +ĠngOn Init +b p +ĠGold en +AC HE +Ġwor ried +az i +E ar +T ake +(f p +bur gh +_ Data +g res +ĠO nt +p us +Ġtrans parent +Ġp ocket +Ġr am +igr ations +. čĊčĊ +Ġ[ ( +Ġadopt ed +Ġreported ly +ĠD ream +Ġ} ));Ċ +los ing +Ġte eth +ĠBook s +", & +enn y +LE MENT +Ġg el +ĠPl ant +! âĢĿ +.h ost +ĠRep ly +re ngth +Ġrecogn ition +Ġ}} >Ċ +L A +Ġmir ror +Ġassist ant +( device +Ġspirit ual +b uilder + § +Ġou tr +Ġt t +ĠP ER +Ġrad ical +Method s +Ġp ace +ud y +Ġg ut +ĠG reek +Ġnon atomic +ĠP aper +_G PIO +Ġob st +.A d +viron ments +ĠS ov +( con +ĠTrans action +. assign +ĉc atch +el ter +Ġbit coin +_G R +ĠčĊ +met ic +Ġtrans formation +åı · +Ġr gb +istrib utions +Ġimp licit +/ in +dest ination +аÑĤ ÑĮ +Z ero +Ġun set +. where +.g o +Ġform ation +Ġdeclar ation +() čĊčĊ +ĠEx pl +ĉĉĉ ĠĠ +/ pro +.J SON +Ġdes k +.sub str +//---------------------------------------------------------------- ------------ +ly n +p son +dis able +ĠF unc +ĉ Assert +ĠM ARK +Ġdefe at +Ġbl ind +Ġconst ants +. headers +UIL D +Ġexp enses +P ixel +Ġh r +Ġf el +ĠEast ern +_d el +ĠC ub +Ġs q +ĉc ount +ĠD irectory +Ġex clus +Ġhistor ic +Ġ ------------------------------------------------ +Ġcom position +Ġdata GridView +ĠB urn +ĠB C +M aster +Ġsp awn +Ġbe aring +.Set Active +il o +Ġg allery +Ġfound ed +Ġav ailability +.s qrt +Ġp es +ĠD OM +m ate +O ct +Ġmatch ed +it ivity +Ġan xiety +.pr ice +ĠIn stant +ì Ĭ +Ġt ut +IC ollection +.sh ared +_s ql +t bl +lib rary +_de stroy +erm al +ĠNot es +ĠE in +Ġsou thern +ĠOTHER WISE +Ġmac ro +.l ower +cl s +Content View +.l ink +const ant +ĠB es +Ġsome body +n b +"> { +( local +.. ... +ĠN ull +m x +Ġà § +Ġp ause +-------- --- +_M O +ĠC M +Ġfor Key +ĠD VD +Ġclose st +_DE VICE +ĠSte phen +ĠB BC +ĠTr avel +P aint +ĠResult s +ĠR ule +Ġt p +Ġrat ings +c in +c sv +> / +ĠG OP +l ad +Ġ ÑĢ +Ġindex Path +m atrix += f +ars ed +Ġ} ); +ĠC os +ĠS core +Ġt ak +ĠE SP +ĠIN C +_N ULL +-f lex +"] [ +int o +el and +Author ization +_F ALSE +Ġg ate +Ġv id +ist ent +T IME +Ġre write +Ġt ie +Ġarch ive +.event s +.get Parameter +ĠPer mission +Ġprogram me +Ġ é +j ud +Ġcam eras +(s ys +ĠSy rian +Ġimpro vements +Ġh ip +Ġsu icide +Ġsch olar +Ġcompat ible +rem ote +.d own +F UNCTION +Ġman aging +ĠUI Kit +. raw +>> >> +Ġdem ands +ell ite +Ġd ent +ĠM icro +åı ĸ +'] [$ +ĠI E +im ension +Ġt rem +Ġg ained +.w ith +. ok +h ou +Ġb om +amp aign +Ġjoin ing +f ish +Ġadd Subview +Ġnor thern +.c or +ore t +D ie +in ish +_com p +Ġatt ended +Ġcoll apse +ĠS S +ac ent +_E QUAL +ĠDe ep +R GB +ĉ test +ol ves +us et +Un ityEngine +w riter +Res olver +, % +if ference +_re move +ond a +Ġfem me +de code +Br anch +Ġfl ush +Ġinnov ative +Test s +Ġ[' ./ +Ġcover ing +. admin +ultip art +(l ambda + namespace +ĠS port +Ġ! ( +ac les +Ġde pression +ĠK ong +Ġp ert +ĠCon n +ĠOther wise +/ home +s upported +Ġp ink +Ġinv ited +ñ os +_en abled +Ġ- Ċ +F W +en ers +ĠM Y +Ġsuggest ions +Can vas +Ġf er +ĠMarket ing +@ Test +unt u +ĠV en +ĠC ou +iv als +Don ald +lim ited +ĉĉĉĉĉĉ Ċ +Ġanal yst +( entry +Ġrepresent ative +_at tributes +Ġf ur +.h ide +res p +ado res +rid es +ĠJ osh +ro bot +ĠN AT +Ġs esso +Ġintegr ated +: true +part s +Ġst upid +: event +@end section +Ġp u +.T able +ĠY ii +` ;ĊĊ +Ġcl ang +=" "> +eng an +_param eters +.int ernal +ĠMod ern +Ġmet ric +Ġsem i +={ {Ċ +.am azon +ĠB B +aint y +view port +Ġstart Activity +dis patch +**** * +Ġfl av +iffer ent +[ this +Ġst ake +Ġarg ued +vious ly +.w ork +ĠO ak +O ld +( async +not es +Ġfl ip +Ġdis ag +ĠT E +ĉ error +< ' +Ġ» ĊĊ +Ġfilter ed +ĠM ach +Ġh ung +_d ump +_s amples +-dis miss +Ġr ay +Im plemented +D K +Ġj ed +Ġbreak s +Ġf its +. gr +ĠZ ero +or o +Ġequ ally +Ġ' [ +Ġconcern ing +< meta +play ers +_P OS +_s im +J an +Ġyour s +ĉ N +Ġsp ir +Ġch ampion +ĠAn alysis +ap a +ĠNS Log +_l ines +ñ a +ĉĉ ĠĠĠĠĠĠĠ +.S c +Re p +etro it +ur able +M IT +com pat +own ed +_ind ices +], čĊ +Ġdis covery +ĠDie go +ob i +. Index +Ġtrend s +PL AY +.n o +Ġl ens +_c fg +Ġan no +ag an +Ġperiod s +ter ms +y z +Ġattack ed +ib ration +PEC IAL +_ grad +Ġaccord ance +.Read Line +.de vice +ri x +. container +m ay +erc ise +ĠL u +Ġr g +ĠÑģ ÑĤ +ĉĉĊ ĉĉĊ +( un +TERN AL +Ġless ons +Ġalleg ations +Ġtrans mission +.Re f +M obile +ĠT ournament +ĠN ut +ĠG a +ĠCap ital +def inition +- exp +c lean +Ġfant asy +Ġenh ance +ent ence +'] :Ċ +ack ets +Ġcelebr ate +@ ", +Serialize Field +Ġarray s +t b +ĉ st +[ assembly +( reg +.c ategory +Ġimpro ving +Ġsal ope +Byte Array +Or iginal +Ġ[ {Ċ +åĽ ŀ +ĠCl in +oen ix +ĠS amsung +Ġmaint ained +Ġag enda +f ail +Ġpres ents +Ġtim ing +.m ark +' >< +Ġprom ot +Ġin cl +_ only +ë¥ ¼ +ĠAtt orney +- date +Ġlands cape +Ġf u +S Y +.p rop +ĠA rr +p ag +Parallel Group +': čĊ +Ġlog s +a unch +unc i +n ama +Table Cell +iss ues +. { +ec urity +_ex ec +old s +Ġhost s +Ġpro to +_ import +_s ort +ĠB ow +ĠN ormal +ĠF arm +.create ParallelGroup +R otation +. err +Ġp leased +it age +.W h +ĉĉ ĠĠĠĠ +M R +ĠM ORE +ĠN atural +_ transform +B ASE +ener al +ut down +.common s +W T +Ġa an +. Result +d og +Ġclick ing +), ĊĊ +# line +Oper ator +Ġc iv +Ġm erg +ob uf +ng then +Ġ[ { +Ġcan cell +tr igger +. : +W ORK +decl are +Ġdecre ase +ÅĽ ci +lo om +.N one +ĠM I +ĠJ ason +Ġhealth care +iam ond +s ylvania +* x +ĠR a +[ b +Ġprint ing +ph abet +ĠLab our +op per +Ġz ijn +-t arget +_F UNCTION +Ġo ct +ени Ñı +åľ ¨ +Ġwest ern +Ġcomput ers +ĠR ET +Hash Map +[ String +get Value +_D ATE +.N ext +ĠF if +é l +ick ed +æ İ +-M M +Ġ{ ĊĊĊ +Ġcontact s +Ġdig its +Pro du +Ġunus ual +Ġrapid ly +t ures +Ġang ry +c ancel +xx xx +_p arser +id ity +_P REFIX +Ġme hr +Ġrare ly +et he +op es +Ġ% . +work s +Ġthe ta +Ġcontrib ution +ĠT ony +Ġsqu ad +аР¹ +Ġî n +th ere +out ed +ĉ q +Ļ Ĥ +g ood +L I +é¡ µ +ĠL iving +iz abeth +Ġk t +ĠD allas +] ],Ċ +Ġ/ >ĊĊ +Ġrais ing +/r outer +_g ame +ĠC UR +z ens +. es +Ġfont Weight +(f unc +not ification +Ġ'../../ ../ +Ġbl ame +ãĢĤ ĊĊĊĊ +an co +Id entity +f ollow +Ġart s +x s +Ġofficial ly +ĠSt udio +Ġrecommend ations +Ġloc ale +Ġam ateur +ĠEn able +Ġcap s +. End +- add +_g shared +ĠC T +For ce +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĊ +Ġor ange +Ġl p +Ġanswer ed +.G rid +Ġd ual +Ġstrateg ic +Ġnob ody +Ġf atal +_ est +( el +Ġì ł +ĠB udd +A IT +_f actor +- one +ĠH AVE +" čĊčĊ +Pro f +Ġä r +str ings +Ġdir ty +ĠF ace +ĠB egin +ĠB us +Ġw is +åŃ Ĺ +Ġspe aker +Ġcar rier +ĠO m +Ġhad n +All ow +:: __ +Ġver b +ĠCom plete +ĠE asy +Ġb ills +ĠĠ ĊĊ +Vert ical +Ġpr on +ĠDef ine +Ġlook up +variable s +Ġpand as +um es +Ġinn oc +Ġset Up +ĠCh ampionship +art ist +ĠC Type +F oundation +๠Ī +ĠSet up +Ġrec ipes +ĠU IColor +ĠF ight +Ġauthor ized +_c lick +_s uccess +ang an +ĠMount ain +ĠDo ctor +Ġeg g +ĠMedic ine +c les +` .Ċ +[ int +d ashboard +ĠApp ro +-d r +Ġprodu ces +Ġrent al +Ġre load +Ġarr ival +sp ot +Ġund ert +Ġequ ipped +Ġpro ved +Ġcent ers +Ġdef ines +al so +Ġop acity +ĠUn fortunately +ĠIll inois +Ġн е +ĠTem ple +ĠTr ail +ĠK elly +Ġmeasure ment +Ġsepar ated +-c ircle +H ey +ĠRE AD +ig its +Ġ ib +ĠM OD +atter y +аР· +Ġv end +ен ÑĤ +ĠHttp Client +s afe +_A SS +ic it +ĠCon struct +ĠC lo +ĠS ix +_T OKEN +(b lock +Ġwarn ed +/* ! +! Ċ +Ġinnov ation +_ " +Ġ );čĊčĊ +Ġsp ots +Ġcho osing +.c s +Ġflex ible +U Int +Ġscr atch +- al +Ġf estival +Ġout standing +================================ ================ +M ean +ĠO regon +s ymbol +. account +d ney +'' ' +! ", +Ġpart icle +à ĥ +[ MAX +IV ER +ER ENCE +NS Mutable +ĠColum bia +_ ĊĊ +.f r +Ġc ogn +V R +ĠMethod s +ĠM ade +ĠB R +ĠEl se +Ġeg gs +Ġsw ing +ĠIn v +Ġdise ases +Ġf irms +Ġle mma +}` );Ċ +l ings +Ġg ym +umin um +.T rim +M em +Ġcritic ism +ibern ate +_T X +ion i +Ġguid ance +Ġrepeated ly +Ġsup plier +Ġpaint ing +.F ragment +ed Exception +Ġw iring +Ġcour ts +W EB +æľ ī +\ . +ill ance +Ġb rows +ĠP attern +PL ICATION +ĠSum mer +Ch ain +Ġc ute +mer cial +Ġd il +ĠFrank lin +ĉg lobal +IN CLUDING +h istory +Ġl st +Q t +SD L +al ia +i ere +( ... +ĉc in +iff s +vel ope +ĠR oot +cl uster +User Name +ign e +< S +Ġf est +Ġindic ating +ke eper +Ġc ada +é g +cons in +ĠG B +Ġl b +em ony +-icon s +_d oc +Act or +e lem +.De lete +Ġin fection +ĠPriv acy +Ġgreat ly +ĠP os +ĠT reat +Fl ow +Ġattract ive +ĠMar c +s udo +tes y +- an +ab ama +ĠW ould +Ġsu ck +index Path +ĠE t +T imes +Ġclub s +_ass oc +Ġac quired +(" : +Ġint ense +.m aps +Ex pected +T oggle +Ġa y +Ġl ifestyle +-c alled +ĠS now +V olume +Ġcann abis +ĠD irection +ĠLim ited +-s pecific +Ġd owntown +/ icons +Ġre ven +L eg += null +Key board +') ). +Ġ"" ;čĊ +Ġatt itude +.n avigate +- error +AM PLE +ĠJ ay +v r +c ow +.com pile +Ġmem ories +_m ark +ĠMin nesota +Ġk osten +Ġprob ability +w arning +Ġgen etic +F ixture +ĠHash Set +N ombre +_m onth +Æ ° +- start +xy gen +ĉ ft +i agnostics +ĠMat thew +Ġconcept s +Ġcon str +. State +и н +N ov +Î ± +ĠP anel +ä¸ ª +com pare +> ()Ċ +Ġapply ing +Ġprom ised +Ġo x +nc ia +ĠValid ation +ort s +_c ur +e lect +ey e +( Data +Ġreport er +ĠB uff +Ġs r +Ġ" ; +ick y +Ġtemp or +S N +Ġres ident +pi res +ys ical +Ġend orse +ĠS ong +is Empty +le et +_ util +Ġdist ingu +ĠT alk +ĠM ot +( default +.A rg +gorith ms +_ words +im mer +_res et +f amily +W W +Ġsav ings +ĠâĢ Ŀ +_en able +side bar +Run ning +Ġal i +Ġtest im +Ġwarn ings +ĠCh em +ĠEx it +Ġfound er +pect or +Ġr m +_d ataset +ĠD as +Ġh an +Get ty +á l +Ġn y +Ġpo verty +Ġresult ed +.b y +ĠVis it +Ġobt aining +/ '.$ +ĠĠĠĠĠĠĠĠĠĠĠ Ċ +sh all +_LE FT +UI Image +_ Name +h ave +ĠN ob +l r +- footer +Ġn aked +ĠG arden +\F acades +Ġgrad uate +Ġfranch ise +pl ane +Ġcontrib utions +Ġstring With +Ġc rypto +Ġmov ements +ath ers +Ġlif etime +Ġcommunic ate +j ar +ĠFr agment +_ IF +ĠN avy +ĠF igure +Ġsim ulation +_st op +Ġreport ers +Ġvers us +aj a +ĠÎ ± +Ġgovern or +List Item +Ġse aled +.Back ground +ed i +ash ing +Ġl ip +ĠI h +mer ge +Ġn ec +el ocity +ATE G +Ġse eds +Ġflo ating +_F A +w alk +ĉ user +_de pth +Ġw age +@ app +N il +( [" +( vector +Ġsecret ary +Ġj Panel +ve z +³³ ³³ +d irection +ĠE P +Ġh unt +Json Property +ĠP ORT +] ", +аР¿ +ĠFore ign +pan ic +Ġtri als +ĠA le +Ġr ural +- value +author ized +ĠScot land +.d rop +ĠM T +ç ± +row th +File Path +Ġrec all +if le +Ġc el +ĠSE LECT +k n +_c ase +Ġc rop +s ure +p ot +IC S +Ġst em +Ġindust ries +P ut +Ġa ber +road cast +Icon s +) ")Ċ +æĪIJ åĬŁ +g ui +Ġassum ed +Ġr x +E A +è § +EL L +Ġdo se +Ġin e +Ġde eper +l ider +Ġord inary +Ġg olf +_IM AGE +ĠN AME +(m odule +Ġat om +Ġbel t +Ġoff ices +b eta +Ġphilosoph y +( JSON +-f ield +Ġintrodu ce +Ġconven ience +opt im +> "Ċ +ath y +Ġemploy er +qu ate +Ġed ited +Arg uments +ĠN ations +__ ) +Ġno se +ĠS ample +' )ĊĊĊ +Ġc ake +.get Attribute +H D +Mod ified +Ġpredict ed +Å Ħ +an ie +S orry +(d oc +w ind +ie ve +Ġprov isions +AT ER +OT E +M Y +.A utowired +ĠB ath +. Boolean +Ġback end +.M ouse +ater al +p aper +Con st +ĠV R +_ entity +_C TRL +ĠProte ction +ĠG M +ĠStud y +Ġsou p +ot ime +' use +] " +/ users +a ug +ĠH ong +_n orm +ãģ ¨ +Ġse cre +(B uild +ĠCon tract +ol as +Ġsa uce +Ġaggress ive +Ġrac ial +char acter +@ @ +Ġcomp ile +ĠV oid +_re m +_m emory +k k +Ġm ic +S ame +U tility +ĠH tml +ĠX ml +Read y +Ġg all +Ġalleged ly +ĉĉĉĉ ĠĠĠ +ĠMet al +ĠPerson al +Ġborder Radius +rx js +object s +Ġwant ing +Ġb owl +v endor +offset of +ĠR s +ĠR ating +Ġr ally +_N ODE +ĠM ix +Ġadvert is +Ġnarr ative +s al +Ġm c +SE rror +Ġf ingers +Ġaccom pany +Ġt ired +Ġstr ide +Ġgu i +el ist +Loc ale +Ġrele ases +ik ing +Ġan ger +)) )ĊĊ +alle st +Sum mary +( O +(f or +Ġbasket ball +Ġroad s +ĠInst all +ĠF ab +it map +Ġ) )Ċ +Ġinter section +ighb or +ĠB ry +ĠHER E +So ftware +elf are +ac s +Ġtrail er +.get Class +ch ars +Ġreg ulation +Ġref ers +Ġde struction +Ġcontin uous +ĠAust in +é ¢ +ak an +.w indow +ĠTem plates +Ġabs ence +: n +Ġdis order +fl ash +Ġde let +bo ards +ĠĠ ĉ +RO P +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġac qu +Ġlaws uit +ĠRe views +Ġgar age +t imer +Ġe j +ĠRect angle +Ġflow ers +il st +ĠIn stance +S uper +d et +dis posing +ĠE S +ĠI C +ver e +S k +_ch annels +put ed +/ null +nn en +ĠG allery +_g lobal +Auth entication +ĠR ank +Ġblock ed +Ġcal m +mark et +ĉ val +Ġa ug +per iod +ĠCon stant +Ġ?> ">Ċ +Ġl obby +p al +Ġs ink +ia h +Ð ¡ +urn ame +Ġcon ver +Ġinvestig ate +Ch rist +H ub +ĠIN D +ĠP ed +ur as +ĉ url +ĠT ro +Ġpre ferences +Ġguarante ed +` ĊĊ +Ġport ions +Ġeval u +' > ;ĊĊ +.AutoScale Mode +Ġc ats +Ġreg istry +ul us +F I +p ayload +- search +Ġstay ing +ac ious +Dec oration +Re view +In f +Ke ep +it is +, String +Co ord +Ġper o +S ex +ĠAtl anta +uest a +Arg b +> * +} _ +F ooter +Ġemploy ed +_b ound +v ide +.f unc +$ scope +Ġsp o +ĠAn al +ounc ed +ar ound +Ġrestr iction +Ġsh ops +å Ģ +ĠLat in +-c ol +Ġbare ly +ĠE uro +E r +Ġfa ire +_d istance +_un lock +Qu ote +IV ATE +Ġå Ī +Ġaim ed +ĠRet rie +. iter +Ġwr apped +Ġagre ements +str ument +( product +Ġstud ied +.set Value +Ġy e +ĠC ache +MB OL +Ġquarter back +Ġsy ntax +.getElements By +.v ersion +we bsite +Run ner +_s ingle +at iv +ĠAl tern +ĠBeaut iful +right arrow +Ġd iversity +pl ash +( co +.F ill +Ġtyp ing +Ġcl ar +H it +O O +ac co +w orth +Ġscript s +ĠMuslim s +ĠL L +erv ing +( boolean +Ġbase ball +ĠC AN +MA IL +de pend +Ġrespect ive +Ġconst expr +.* ;ĊĊ +'] ))Ċ +Ġy ard +Ġident ical +if ecycle +US H +up iter +. validate +cl i +IST ER +Ind icator +F ail +Ġdemocr acy +. var +Ġsatisf ied +------------ - +enc er +h or +Ġr ounds +DA O +o a +Ġfl ask += c +[ ]Ċ +/d ist +Ġpart e +Ġconfirm ation +er on +aw are + +Ġdepend encies +ĠV ideos +- row +Ġ** /Ċ +Ġn ou +Ġh over +æ ŀ +Ġn in +ĠUS D +M ac +_L oad +Ġout comes +_s ocket +Ġqu eries +w m +Ġhit ting +in ux +M ich +ud ge +AT AB +Ġvulner able +ä ¾ +Ġport folio +: YES +ĉm ap +B ound +Ġiter ation +in cess +Ġact ors +ĠQ ual +_c lean +ãĢij ãĢIJ +MS G +G reen +ĠOff icer +Ġsm oking +> ', +ĠF lo +++ ; +oly gon +Ġbul k +Ġdr ama +Ġexception s +os ed +Ġ+ čĊ +Ġleg acy +C V +Ġcontrib uted +ĠTer ms +Ġb t +Ġunt uk +Ġal ien +=== Ċ +ĉ Vector +Ġl s +On line +.f acebook +num eric +ock ets +A ut +b ury +-re dux +ĠRed istributions +GLOBAL S +urrenc ies +Ġt ons +âĢĻ , +Ġà ª +(c ol +ĠS ymbol +Ġstay ed +ĠM L +Ġm unicip +Ġsex o +S en +n r +Ġg ains +Ġshort ly +.M enu +à ½ +KN OWN +Ġoper ators +- V +ĠPat rick +/ add +_C O +ir ation +(p ost +Post s +/ _ +Ġpl ug +Ġintellect ual +Ġmet ab +Ġpregn ancy +ĠPrem ier +n m +Ġpred iction +ĠMin istry +Th ree +val uate +ĠMin i +b u +оР· +< ul +Ġd d +ol ving +ĠC ut +Ġs chem +.tr ain +it ate +Ġr ice +Ġbird s +ãģ « +m iddle +struction s +Ġn erv +a que +Ġfl u +Ġsurv ival +ĠGal axy +ĠF ant +. Order +At trib +irt s +é c +M ovie +Ġcon ce +qu arters +Ġm ood +.Add Range +Ġres olved +ãĥ Ī +Ġburn ing +ĉĉĉĉ čĊ +ĠW E +Ġhost ing +L AB +Ġman agers +Ġstre ngthen +< const +ĠFire base +on ed +ĠJ ean +' ";čĊ +ĠS av +.B old +Ġen ables +ĉt mp +Ġman ually +ĠS qu +user id +.f unction +.c ache +LO PT +.S ervices +dd it +t im +< img +ĠTh ings +ĠEvery thing +Ġa pt +em and +Ġroll ing +ë ¦ +. level +Ġst om +ĠW inter +Ġview ing +( values +ocom plete +v ia +up o +Ġabort ion +i ère +ï¼ ij +_B UTTON +_d omain +Ġb ra +ĠA st +in as +Ġstat ist +c od +L R +Ġdr ives +Ġfollow ers +Ġall ies +ĉc urrent +ecess ary +Ġdam aged +_ pt +and les +oun tries +Ġsim ult +e u +Ġcontrovers ial +_G ROUP +Ġr ib +. Info +: mm +.n ormal +_ADD RESS +Ġ íķ +add le +ĠD ur +. Element +W arnings +Ġcred its +Ġin hib +Ġem issions +Ġh az +.y outube +ugg ed +Ġbo ther +ĠK ansas +ĠF ixed +ĠTest s +ĠF IX +Un iform +Ġk ont +>> > +st ation +lo re +at ype +ish op +/ **************************************************************** +Com boBox +Ġvac ation +Ġiniti ative +Ġdefault Value +con cat +ĠK h +ĠW elcome +ized Name +M igration +Ġgrad ient +H ot +Ġhard ly +el o +ĠStud ents +Ġlo ose +at z +.S end +' / +Ġunivers al +Ġenter prise +Ġreg ex +Ġvis itor +ĠF ly +Se q +ภĻ +ĠVis ual +Ġlib raries +ato es +P ayment +Ġp ent +Ġgather ed +VRT X +ĠD M +S plit +Ġlet ting +Ð Ŀ +_error s +ep och +P ARAM +c u +ÑģÑĤ в +ol utions +Edit ing +font s +Ġalloc ated +ĠB ased +( Y +ĠJud ge +Ġbro thers +FILE S +ç o +w b +_P I +' ^ +Ġs word +.s ervices +Ġn l +T im +ig g +ĠMo ore +Ġcrypt oc +åĩ º +_post s +ot ate +? ' +... .ĊĊ +Ġk l +=" $ +Ġdec oration +Ạ¡ +ĠD IRECT +G UI +) =>{Ċ +Ġnews letter +Ġprec is +(p oint +ĠEqu ipment +ut y +ĠD ave +Ġparticip ation +u arios +x it +.A s +ET ER +or ous +Ġsh ield +[] > +ilit ary +. origin +Ġprom otion +U nt +Ġc t +TR A +View Holder +Ġsig ma +d elta +are house +con tract +( Vector +Ġcompet e +/ form +/ components +Ġn r +ĠInd ones +Ġо ÑĤ +ĠV olume +.f iles +(res p +/ models +Ġsur f +stand ard +/ o +ĠXCT Assert +V ICES +.C ode +SE D +Ġact ivate +D elta +Ġlimit ation +ri j +Ġpregn ant +: ^( +Ġs our +p ie +Ġexp ense +ic ation +ĠL arge +Ġ ± +ĠB owl +(model s +/ N +P a +.re load +Ġwonder ing +Exec ution +ĉ ĠĠĠĠĠĠ +ĠG raphics +ĠCont in +_j ob +Ġget Name +ĠM agn +ĠD WORD +m ad +Ġn h +fe atures +} ");Ċ +he ets +(tr ain +z n +Ġrecru it +.con nection +Ġbar rel +Ġste am +_set ting +Ġang ular +ane ously +Ġb il +ĠN orm +(! $ +ib t +% ( +Ġpos it +ĠF ather +int endo +L ive +Ġport s +Ġme j +Ġland ing +pon der +Ġc od +_HE ADER +.M argin +Ġball s +Ġdiscuss ions +Ġbl end +H ex +Ġfarm ers +Ġmaint aining +ĠĠĠ čĊ +s yn +[ T +r us +uff ers +Ġcontrib utors +_s ys +.De bug +Ġconstruct ed +om es +? id +sl ider +Ġsup pliers +scri ber +p es +Ð ŀ +": čĊ +\ Controller +)) ĊĊĊ +Ġl ua +M ulti +EN S +S rc +Ġpet ition +Ġsl ave +look ing +V ERT +ĉ vector +S pecial +h h +an ne +ĠN iger +/ views +z ing +end ant +< C +s peed +Ġ{ };ĊĊ +Begin Init +Ġf open +@ RequestMapping +End Init +Ġp unch +S ender +é Ķ +get Message +/t ypes +.P I +(' ');Ċ +oc used +( all +Ġdrop down +). __ +ĠV in +.Fore ignKey +can f +ou red +ĠOrgan ization +ĠÐ ° +ĠC ulture +(cl s +, _ +rg ba +ìĿ ĺ +.data GridView +Ġdo zen +ĠG es +_sh ared +n ick +Ġh osp +om eter +Ġclaim ing +ib les +ri k +æĺ ¯ +en ario +Ġd engan +ob b +m ont +_r ank +('/ ', +Ġap olog +P s +_p ower +ĠG ree +Ġful fill +Ġfire base +Ġf are +ĠH im +Ġbe an +âĢ¦ . +ĠS PI +_R X +Ġper ception +rel ative +comp ile +u um +ut os +a uc +ĠAs k +Ġindic ator +/ th +.set String +ĠWis consin +.D omain +Ġart ificial +De velop +ĠSar ah +Ġl ying +( search +ĠEmp ire +urr ing +æŶ éĹ´ +=" ${ +Ġget Id +ĠP ayment +trans ition +Ġ ]. +ix in +V T +- select +Ġdemonstr ated +Ġlast Name +employ ment +.get Property +Ġf ought +file Name +ĠP ers +-c ard +a str +attr s +Ġprom inent +Des ign +anc ouver +ãģĹ ãģ +ard o +se cret +Ġr ag +Ġpo ison +-m an +, omitempty +ĉ un +it zer +ĠCas ino +ĠR oss +- foot +(result s +Pl an +Ġlas er +ê¸ ° +_D R +F acebook +Ġbo ards +st a +] ], +Ġt iles +S IZE +Ġ= ~ +Ġprem ier +oc ab +Ġenc oded +Ġres erve +ĠAfghan istan +ĠList Node +url s +Ġsub mission +Ġne u +Ġ# +# +_P OST +Ġmo ist +ell i +ellig ent +. alert +ó d +b re +ĠCol lect +Ġgraph ic +Ġlong itude +ĠPro vid +ĠCal culate +x ffff +c riteria +Ġw aters +ro ck +lo quent +ĠT rib +Ġbur st +Ġsuff ix +.Ext ensions +ish es +iv el +ĠLI KE +ĠGet ty +.Action Event +.s lf +ĠH AL +up al +E AR +ud i +_time out +U F +ĠSing apore +ĠAd vent +_int erval +cha ft +ĠE mer +Ġtele phone +ĠTur k +_ interface +ĠO wn +Ġencour aged +< Object +_T ext +ĠOnt ario +ĠApp ly +.f irebase +Ġant ib +P riority +ene z +D ays +c id +urre nce +; / +inn ed +Ñģ Ñı +Ġve z +f w +// $ +att ack +Ġstart up +ain ers +.f ragment +op acity +( conn +he im +.n etwork +( stream +ĠN ON +t ol +ĠX box +ĠD S +Ġc ached +Ġprostit utas +ĠB alt +(' [ +Ġno except +" ' +Ġs d +. valid +_ ag +Ġr aces +Ġro d +itud es +< >( +.Pro duct +Form s +NE W +P ay +ĉ boolean +_ contact +ĠElect ric +sk ip +Ġw ur +Ġch ronic +_d river +ĠS ab +ĠU lt +ĠR ad +ST ATUS +ĠLew is +O B +Ġgift s +.Re c +TR UE +Ġint ensity +Mark er +.com pare +ff ic +C ookie +ĠB aby +ĠBig Decimal +ile t +ĠHOLD ERS +ĠL ady +Ġl ung +ĠAl abama +Ġd ess +` );Ċ +ĠB uilder +_reg ion +Ġne utral +Bo th +Ġh p +Ġh orn +Ġseg ments +ĠE C +"=> " +( rec +ĠP i +G M +Ġl aptop +Sc alar +is d +-d ialog +ĠAnd erson +Ġmist akes +ĠH an +j es +est ination +Ġprom ises +b id +ĠSc ient +G IN +ĠPer formance +b age +. users +le ading +Ġor al +G raphics +_P TR +h ang +Ġin ev +process ing +F actor +ĠN A +$ string +Ġground s +.Save Changes +c lock +cri pcion +ĠNew ton +g c +.in cludes +Ġbl ast +Ġ'- ' +Ġpued e +.S ession +Ġgre p +_f inal +ĠG ay +ĠG ive +ir i +-st ar +ĠUI Image +_ep och +ub b +ent h +Ġel ite +Ġcampaign s +ĠP orno +_ assign +Prot ocol +ĠBe ing +ĠAir port +Ġconvent ional +ĠW at +ĠC I +ET A +ĠAnth ony +Ġtable t +( format +Ġconsist ently +ĠI owa +Ġav atar +.c ursor +! [ +Ġh anging +H er +S uch +';ĊĊ Ċ +orge ous +() == +Ġview Model +Ġ ãĥ +Ġel s +ĠAg ent +F etch +ap or +Ġc x +p read +ĠP ier +oe ff +S n +ĠV irtual +A pr +.Wh ite +_M OD +ĠPoint s +å¤ ± +Ġgen es +Ġv endor +Ġmain stream +< src +ĠEl izabeth +Dec oder +- state +ĠG lass +nc y +adi ans +_m on +ĠRem ote +Ġwire less +ĠM i +å ī +è¡ ¨ +st age +ĠT ile +ll ib +V ariant +== Ċ +Ġgold en +(Q String +.put Extra +ĠD om +ĠAn imation +Ġinter active +if act +éĻ ¤ +LE T +Ġfrequ ent +Ġ< >Ċ +F ilename +Ġs ne +ĠFoot ball +Ġr ival +Ġdis aster +ion ic +ĠD amage +. Resource +- en +ĠT ypes +get String +( board +Ġb ol +pl ain +z ym +ภ² +Ġsc anner +ild er +_msg s +æ ı +(int ent +Ġde struct +Ġb ust +ĠE mploy +on i +ĠUI ViewController +Ġodd s +ear er +Ge ometry +Ġy ii +_EX PORT +ĠAtt ack +Ġn iet +Ġim pression +ĠG il +_pro b +ĠC F +ĠEx perience +/pl ugins +.M ethod +Ġbelie fs +N ative +_b uild +Ġv ig +Ġr anks +cover ed +s uch +G uard +.p ack +add er +iv ia +l ng +Ġв Ñĭ +T imestamp +_n ow +Ġp oker +Ġun c +Ġsh apes +-t ypes +_per iod +p k +Ġveter an +Ġson o +Ġappoint ed +over flow +.d river +_c at +ut t +pl ant +im b +ĠAc cept +Ġconc ert +ĉ node +ĉ z +? >čĊ +Ġb anned +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġto xic +Ġdisap pe +È Ľ +Ġgr ace +ate ful +Re ply +ĠCru z +Ġsc rap +Ġkey words +s imp +Ġmort gage +Ġcy ber +ĠEx ecute +Ġlat itude +if u +.C OM +d bo +Ġsort s +ĠG as +om ial +.L ocal +Cell s +.Re place +String s +.f it +ĠTh ird +% ",Ċ +Ġ{} ". +ĠS ony +Ġ[ : +Ġfall en +. ')Ċ +in h +ĠM C +Ġred is +C odes +Ġprofile s +h ook +Reduc er +_F UNC +Ġn avigate +str len +Ġh orm +á ŀ +ĠS R +. boot +Ġdig est +ĉ header +.find One +æ ģ +Db Type +n ia +_m erge +Ġdon ne +/ Getty +_CH AR +Ġb ands +. URL +art ial +Ġf req +Ġs ist +N g +Ġrender ing +\ Core +Widget s +ĠV A +Ġactiv ists +St e += _ +all a +St amp +Ġload s +Ġx x +ĠL earning +.M vc +u ir +(" $ +Ġconnect ing +Read Only +ur u +ĠE ag +B IT +_DE L +å § +arr ass +ext ernal +ĠY OUR +ĠB rew +ĠF ive +Ġres ize +ig id +er ation +ĠÑ į +åĬ ł +ĠC atch +Ù ģ +ĠLe on +am il +.B ody +Cl ip +/ list +.b r +Edit Text +ĉ db +.G ame +(Build Context +back end +.R ed +face book +.url s +m r +rol led +---- --- +Ġinter vention +Ġretire ment +ĠK it +ĠP RE +Upper Case +ĠS ocket +Ġ: - +Ġstudy ing +ĠMet ro +ard ed +Ġconvers ations +C alled +Ġexam ine +ert ificate +.g z +-res ponsive +Ġref und +_n etwork +allow ed +em pt +Ġme als +C ategories +Ġtravel ing +Ġk g +Ġsh ame +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġexplicit ly +Ġmath ematic +ĠS uite +ĠR GB +****** / +Ġmix ture +lear ning +.t emplate +att s +w x +ĉ ctx +.p roperties +Ġdrink s +ĠE ither +set Text +.get Data +.z ip +Ġreve als +< table +.Hash Map +ĠH ur +) ");Ċ +.f ramework +ĠST ART +feed back +Ġsaf ely +. icon +config ure +. lock +.l ayers +/> .Ċ +Ġrank ed +_ impl +ĠHand les +Ġhost ed +Ġup dating +al bum +é Ŀ +Ġsh ader +Edit ors +- round +[] { +Ġse p +ĠH i +TE M +look up +.m an +_IN PUT +Ġthreat ened +_IM PORT +Ġd rops +ru it +s id +bo th +ĠEx cel +Ġj er +ord inary +еР¹ +V IEW +re ply +Ġ) :Ċ +color s +ver ified +_T r +_p arse +Ġcon gress +P romise +int s +ĠM other +.A pi +ĠD uration +Ġfirst Name +inherit doc +ĠM ars +Ġa pr +OD Y +Ġvis its +Ġhe aling +let ters +)) );čĊ +f uture +.F ramework +Ġk iss +Ġinv olve +Ġsil ent +ad ows +Ġany body +s ch +Ġsole ly +- img +Ġprop ri +Ġin struct +Ġlic enses +Ġm eth +Ġcond em +ĠD omain +ĠHarr is +Ġs Ã¥ +CE PT +B atch +@ extends +ĠCONTR IBUT +.Data Frame +_p acket +rec ision +Ġfoc using +. ht +__ ":Ċ +: Get +ĠK C +Ġpass age +Seg ment +_c enter +-z A +_B L +Ġconv in +Ġclass ified +ĠNS Mutable +_ ap +t ile +Rect angle +(n ums +v ens +ĠUI Button +ĠF eder +am o +Ġout line +ĠPar ser +Ġâ ī +ĠWork s +.S chema +Ġeng ines +_com mon +_ old +Ġset ContentView +Ġ/// < +ĠB T +f m +Ġd ivers +_ weights +em ark +ĠA CT +Ġpro portion +over lay +.dir name +ĠG it +_REF ERENCE +< > +l b +_r ule +è´ ¥ +ĠPut in +Ġsleep ing +() :čĊ +Ġpres erve +Ġpar liament +ĠLook ing +Ġpick ing +ĠDis patch +Ġsl ip +ë ĵ +ĠL yn +_sign al +config uration +ĠP itt +ad en +pro cedure +Ġenthus i +f ight +ĠCons ider +Ġt orn +Conn ected +.c os +_group s +ĠTh ink +Ġdel iber +Ġres id +work ing +.column s +ĠCal led +Ġes lint +> ", +_D OWN +h ist +ĠAdv anced +Ġre wards +act ors +Ġsil ence +Ġmy th +Ġne ur +Ġa uction +.Get String +ek s +( project +ĉ msg +ĉ output +Ġcomplaint s +, S +Ġt bl +Ġ, ĊĊ +ri ors +ah ren +Ġlawy ers +re dux +_s ymbol +off ee +_RES ULT +( Name +UT C +.current Time +Ġorgan is +. arg +Ġmin im +w ick +Ġrece ives +B alance +Ġspeak s +ĠD ays +ĠBel ow +t ipo +P resent +Ġres erv +h p +Ġr it +_R IGHT +-- ) +Ġchair man +D IS +ĠBO OST +Ġexper iments +__ );Ċ +Ġst amp +Ġf ert +Ġf ond +T er +el ve +ure n ++ i +end ency +Ġvirt ually +... " +ï½ ŀ +- cent +_un ique +Ġpr icing +m ic +RES H +Ġ:: : +Ġan notation +ĠC ircle +ong odb +it as +Ġ% ( +( component +Ġо б +( port +-h our +. obj +L BL +Ġj ury +GB T +Ġsp y +ĠProf essional +Ġ"" ;ĊĊ +Ġstri king +Ġdiscrim ination +Ġp ays +lic t +ent es +Ġthrow ing +ĠPl ugin +( def +ĠRuntime Exception +ĠM igration +Ġd ic +b ag +on ia +Ġcor ruption +( Map +Ġpr z +.d to +Ġac quire +State ToProps +Ġlo ving +оР¶ +_p attern +Ġemot ions +Ġpublish er +_b e +Ġcoup les +o j +ĠCh art +Ġt rop +.t ool +Ġestablish ment +Ġd ol +Ġto wer +Ġl ane +ĠSy dney +Ġfill ing +claim ed +Ġdialog ue +Ġcon vention +book ing +pare ncy +æ ± +ĠGener ic +\ Schema +Ġr anges +/ ch +Ġpan els +Ġr uled +çĶ Ł +.t s +_s ets +Ġclean up +Pre vious +ĠAn imal +($ ( +ĠA ve +oll ar +_e val +ĉ Name +(t ree +Ġ" ] +Ġdut ies +=' / +Click ed +Ġdifferent ly +ĠCl ark +Ġd it +olog ists +Ġsy nd +Ġs ends +- known +k b +ĠMod al +it ative +Ġr acing +Ġhigh lights +ĠSim on +ĠCapt ain +ä¿ ¡ +ĠC B +cont in +ar an +Ġphys ics +ret ty +et al +.m d +ax ios +Ġspeak ers +Ġpre p +Ġaward ed +ì§ Ģ +ĠC orn +ĠN ature +UD IO +Ġpro j +- pre +[ u +Fe atures +Ġis Equal +B inary +s ig +Ġconf usion +ĠH at +Ġkt ó +.config ure +M ON +/ edit +_A dd +, true +Ġc li +Error Message +- loader +Dim ensions +ultip ly +Ġ{ !! +ĠSql Command +Ġsp oken +Ġp ics +Ġto y +( Key +ĠLo op +Ø ¨ +E ATURE +in ction +_set up +w rapper +Ġt ong +c ular +O pt +.P l +=" , +(l ength +um n +Ġch rom +Ġse vent +ĠIllegal ArgumentException +ĉ start +Ġbeg un +CE PTION +dat aset +ĠF ailed +col s +Ġkne e +im ore +.sp lice +sh ell +ig gers +Ġthem es +ĠD J +ĠAss istant +- $ +May be +Ġorder ing +ĠInt elligence +ĠMass achusetts +Ġfail ing +el son +G reat += i +.re st +Ġinv ite +-dis able +.Group Box +âĢĻ est +Ġtack le +g v +et ter +Ġ), čĊ +_r ules +.w arn +function s +ĠChrist ians +Ġback ed +Ġsl ider +Ġenjoy ing +n est +Ġh ij +_m s +// * +An notations +ĠVariable s +< V +( server +ĠOr acle +element s +Ġorgan isation +_point er +ĠHe aders +[ d +Ġdead line +iss a +Ġkn ife +ĠNAS A +ĠHe ight +ĠAs ync +Ġven ue +.d om +bour ne +ĠHaw ai +Ġmem o +ict ions +Ġsurve illance +om i +/ assets +Ġed u +Ä Ľ +Ġro ster +Ġh ired +ĠT ok +Ġpl acement +ur ations +Ġset State +ĠMag azine +Ġhor ror +T ry +Ġl ag +ĠEvery one +th ur +)) ;čĊčĊ +. return +Ġsy mp +âĸĪ âĸĪ +Ġn ights +work er +Ġa le +ennes see +.st ep +Ġsynchron ized +our i +Do es +. change +f on +.set Background +irc ular ++ - +ĠC IA +ĠJ ane +ĠSim ilar +- I +level and +Ġpros pect +_f ound +ĉc olor +.D iagnostics +Ġann ounce +Ġassum es +/ tr +Ġb d +ĠCar bon +Ġanal ys +.de st +n ik +ĠL ie +- index +Draw able +ĠT AG +Ġtri angle +_F LOAT +ĉĉ ĠĠĠĠĠ +.bl ack +v ue +cur acy +Ġaffect s +Ġsure ly +Sl ider +uk i +c ery +Ġun ter +.pro file +ord on +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +le ave +Ġsmart phone +g ie +Ġcons pir +Ġt utorial +ç± » +Ġc ab +ĠSum mary +* ĊĊ +ä h +" This +Ġsl ides +" +c ycle +ĠB ull +path s +Ġun p +Ġview DidLoad +_M odel +Ġassert True +Ġr ated +De cl +vert ed +ĠD at +b rew +Ġpoint ing +M s +ĠPoint er +) ' +_n on +ĠSE C +Ġy eah +g ency +initial ize +f ly +[ pos +, g +Te le +Ġj oke +Ġcl ause +.find ById +en es +( instance + £ +Ġs lic +_h ome +Ġ*/ }Ċ +_p ages +(s ervice +R P +ĠAm ong +.get Current +ãĤ ¹ +Ġs lee += [Ċ +ol er +Ġlib ert +Ġ` Ċ +Ġw enn +l ated +Ġimm une +( Node +ĠPro blem +ĠA bs +log s +Ġ ../ +ĠA DC +Ġ}} ">Ċ +> ');Ċ += b +ĠW ind +lah oma +Ġalloc ate +or ian +Ġpres cription +- quality +ĠMay or +in ely +end foreach +ĠCom plex +k om +T Y +] ]. +. Style +_m any +',' $ +Ġbar rier +ĠF etch +ĠMar vel +Ġres ist +ог о +b idden +ĠRun nable +: false +Ġbuild s +ĠSt age +Ġd ub +emp o +.s ite +;ĊĊ ĊĊ +ĠDen ver +Ġre vel +Ġtrigger ed +Ġd ice +_f ail +Ġg c +ĉ X +ĠTh rowable +.r outer +ĠRev olution +ÑĢ а +_N ON +Ł ¥ +Ġel der +Ġab road +ĠÐ µ +ĠAd ult +bl r +g lyphicon +Ġprom oting +Ġ iz +ĠS olid +_lo ader +ear ly +.en abled +- edit +ĠU L +_ play +ĠInt errupt +Ġadvant ages +uc le +Ġmechan ical +.table LayoutPanel +ĠWork ing +Ġan onymous +R ating +ig ious +_ph one +.addAction Listener +Ġfr an +und en +Ġ*) & +_ bool +ul ative +Ġcon e +ĠM ult +Ġm ö +ĠFor ward +] ):Ċ +Ġconvin ced +act ed +ãģ ĵ +ĠConfig ure +Ġce iling +D er +Ġpass engers +Group s +Ġsoc cer +/ W +avi ors +sw ith +ĠZ one +. Options +ĠM om +ied er +Array s +Ġtreat ments +Ġprotect ing +f ac +Ġpick le +Button Item +Ġblock ing +str ar +à ² +ĠEx port +Ġth rew +ott a +ĠB ASE +.w s +.LE ADING +order By +_d elay +ĠP u +.d ll +ĠCh oose +Pol ice +ĠBE GIN +box es +Ġdiam ond +, l +Ġ ĉĉĉ +Ġcur ious +t v +Ġerot ische +ack ages +ĉ Set +T ick +.b order +static method +Ġch er +in voice +Ġcr u +Ġdef ect +_m etadata +re lation +ik an +[ N +(Q t +( Base +æģ ¯ +be at +ĠEm pty +ĉ o +_sh ift +Ġreg ret +Th ose +C ent +ĠPort ug +ĠIs lands +ĠT IME +Man agement +-s p +ê me +Ġnot ion +un ifu +P K +è¡ Į +ĠCUR LOPT +\" \ +U V +ç º +d ra +c ou += ` +ĠD estroy +r p +.c ancel +G G +r untime +ĠV ue +Ġprogress ive +/s ervices +Ġrun ner +_FR AME +.ToolStrip MenuItem +Ġ' ,' +d elay += utf +Ġscreen ing +Ġpull ing +om as +Ġan th +- new +/ local +Ġi Pad +Ġt witter +Ġd ying +Ġhe aven +ĠU Int +ĠSen ator +Ġpres um +ĠWalk er +Ġover come +ete ction +Ġemb arrass +Ch ina +In clude +RO LL +Ġdata Type +D avid +ภ£ +lo p +-m onth +Ġsc ar +ĠS afe +Ġ **************************************************************** +Ġaccess ories +Ġr amp +_U SE +Ġcontr ad +)) ]Ċ +Ġpre st +ĠH R +ĠR ap +Ġus ize +Ġcap ability +Ġc ort +- next +Ġbur den +_read er +Ġ@ @ +reg ular +ĠK a +M AN +Ġa str +Ġ' ')Ċ +Ġf ed +Ġpars ing +ĠY ears +Ġbro ker +": {" +Ġa kt +In ventory +abe led +Ġarg parse +****** *Ċ +vers ation +Ġc ord +ĠT i +Ġhope fully +Ġa h +ver b +Ġst olen +. Entry +Ġexpect ing +O rientation +Ġpower ed +Ġp ersist +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +'] ); +')) ,Ċ +ĠC ash +ĉ item +gr ades +rop ol +b asic +Ġ" );čĊ +Ġaw ards +(r ange +- all +ĠIB Outlet +ĠInd eed +---------------------------------------------------------------- ------------ +Ġstom ach +Ġfl ower +Ġs ew +_t imes +av is +Q String +ĠR outes +_pro t +Ġcom edy +Ġlog out +Ġwood en +Ġpost er +p iece +.J oin +ĠP ok +cel ona +mut ex +;čĊ čĊčĊ +Ġstri kes +Load ed +) arg +es a +Un ited +E p +PE LL +ĠAtl antic +ul let +app le +Ġsett led +a con +Ġprint er +ĠG C +å® ļ +Ġrender ed +, âĢĻ +he it +s ocial +. ge +ĠR ick +ĠUt ah +g ot +on ical +ĠSc roll +ĠSc iences +Ġj ug +Ġam pl +ent i +LE FT +Ġt abs +Ġenorm ous +.get Key +loc ate +. EX +.st orage +.W e +Ġto ast +ĠAdd itionally +ĠN OW +_ UPDATE +Ġtrans ferred +th a +.D isplay +_ ui +ID EO +Ġmeaning ful +ĠMos cow +, this +ĠVict oria +æĶ ¹ +ĠÐ Ł +.st ack +ĠB arn +pared Statement +: string +Ġb ij +ĠST ATE +Ġemploy ers +ĉ input +( | +Ġle x +in voke +ĉ num +++ , +at ial +ors es +Ġfor k +_t xt +ĠAnton io +Ġ( < +aver se +Ġdev ast +ãĢ Ģ +.D ec +ĠG ard +/ ui +. % +tr i +Ġrol led +Value Pair +itt en +ĠTh er +Ġv rou +ĠFl ow +ĠFin ance +ĠCom b +H C +.set Visible +is l +Ġp k +Ġup set +( raw +ĠV ice +e atures +ĠL ang +Look ing +ĠA ST +Ġtri ps +ĠJust in +b rowser +=" '.$ +. vertices +- co +}/ { +Ġ? , +ĠD omin +ĠBel g +" < +Ġsup pose +add y +Ġwalk s +ERR U +_f ilters +Pre ferred +sc ene +е Ñģ +ĠAff airs +Ġ"# { +Ġon Submit +Ġstock s +/ view +g ree +- get +h it +J o +.get C +Initial ized +ÑĤ и +c uts +( Type +ĠAg reement +ĠViet nam +Ġ/* ! +Ġp izza +- view +_ em +Ġl hs +Ġm uy +ĠId ent +ĠF riends +Ġab und +_A D +.t imestamp +- ' +Ġd uplicate +Ġhun ting +Ġregul atory +ia o +am ous +ĠEnt ertainment +[ A +iat ric +_CL IENT +ĠK ids +/p kg +B reak +)) );ĊĊ +ĠSh ape +Ġrel ating +Int errupt +able Opacity +emb re +Ġmyst ery +Ġjournal ists +rit able +.L ink +Ġstop ping +CRE T +.D B +Ġpopular ity +Ġg ew +Ġim pr +set Value +FL AG +ĉm ax +Ġb ake +w y +ĠEcon omic +Ġen contr +Ġf name +/ de +R ank +Ġbug s +.s m +Ġmed ian +D OWN +ĠS ure +At Index +ĠD ick +Ġ( __ +.d elta +F r +Ġsuggest ing +ĠRec yclerView +, e +ST ART +/************************************************************************ **** +xf ord +Ġrece ipt +CL AIM +read only +Ġeng aging +C a +as ma +Ġens uring +Eng lish +ĠV ancouver +hy th +Ġpurch asing +ĠP I +. word +(s p +.h ome +: def +Ġg ig +ĠV e +for um +ĠM itch +B ay +_F L +Ġs oll +_column s +Ġminor ity +b ird +Ġhand ed +SS L +ST AT +Ġnerv ous +ĥ ½ +Ġfile Path +CRE ATE +A w +Ġp ens +se ed +ĠCom pute +ol k +ĠAs set +re ach +'), čĊ +n avigation +L F +/ util +ĠP ub +Ġâ Ķ +c ion +## Ċ +II I +Tag Name +Ġam id +per mission +if iable +xFFFF FFFF +н и +.B uffer +_ irq +d ark +Ġret val +.f ire +produ ction +.list en +ĠWe ather +Ġbuy ers +. ne +er p +ĠP ent +Ġw elfare +Ġpage Size +ĠSt adium +ert a +Ġle v +amp a +P ager +Ġcharg ing +ĠNet flix +| null +_r andom +.x path +Ġst ere +ĠIS IS +pons es +( loc +ey ond +ĠOff icial +ĠMary land +Data Type +_p ar +{ }, +ĠEn joy +_SH IFT +ĠA wards +_ENT RY +Ġseem ingly +entic ate +Ġheart s +_ ;ĊĊ +ĠH IV +Ġindiv id +ĠFl ag +_ ctrl +ĠC allback +, z +ĠG PU +ĉ obj +ĠPh oenix +ĠB US +Ġrub ber +_A UTH +ĠSol utions +( location +Variable s +.set Enabled +_h igh +W O +G esture +Ġre try +Ġobject ForKey +allow een +Ġm os +ĠC ele +Ġik ke +(c ell +ĠM ODE +ren a +Ġdescri bing +Ġph i +Ġr d +Ġdes erve +Ġwhe els +å¸ Ĥ +Ġcrit ics +N amespace +ĠF ra +Ġ ĊĊĊĊ +Ġall a +Ġrequ iring +æľ Ł +ut ation +Ġdelay ed +Ġadministr ative +Ġb ay +.h idden +T ex +Ġbound aries +Ġ] );ĊĊ +ĠFollow ing +~ / +F i +_con v +_T ITLE +Ġdes de +ICollection View +Ali as +Ġb ite +pat ient +_COMM AND +Com pleted +ĉ elif +( < +B usiness +ĠP ool +Ġpurs ue +ĠB an +_st eps +_DE CL +um ble +Ġcom bo +ĠL ayer +.x r +Ġd up +-------- - +Ġmod ifier +ro b +re z +Ġath letes +Us ed +w ear +Ġlegit imate +Ġ" ĊĊ +Ġh v +St d +ĠH old +Ġsurv iv +ĠAll iance +ĠEar ly +Beh avior +(f ont +/lib s +Ġrect angle +Ġs inger +Ġam p +Equal To +Ġ" ." +Ġgirl friend +å ± +line ar +obs erv +Ġpi ù +Ġcomple ment +With Value +(p assword +t ake +Bl ank +ĠCom par +' ", +_p olicy +m ongoose +_FA ILED +.re port +R atio +.Perform Layout +us able +m ers +_re nder +PE ED +Ġles b +ĉ E +_t ool +Ġl adies +о Ñģ +)) ))Ċ +;; ;; +.d ot +Ġn est +pe ak +uk kit +ec a +_S W +Ġ& ( +ĠOk lahoma +Ġbank ing +ĠN intendo +Ġreprodu ce +_element s +_m ac +pro xy +Ġremark able +}/ ${ +Ġout s +.has Next +M ODE +Ġan ime +.con n +Un ique +D om +Ġimportant ly +itt y +Ġju ice +T w +ĠPart ners +Ġattack ing +Ġport able +am iento +.P ictureBox +.g en +Ġopt imal +Ġre cre +Ġjournal ist +ĠEx tract +ĠMore over +Ġmargin Top +.A p +Ġf iring +Na N +ĉ template +аР´ +. En +Ġdef ence +ĠT el +il en +j an += data +ĠU rl +ĠRe uters +(t otal +ĠFif th +Ġess ays +Ġinterpret ation +Ġchar ity +ĠR ules +Ġsub section +st yled +az er +l ags +L IST +Ġupload ed +Ġtr ash +Ġreg istr +Ġsell er +>' ;čĊ +Ġstart Time +ç Ļ +s y +(Http ServletRequest +Ġtr ap +G C +Ġembed ded +Ġsurround ed +im its +T X +yl inder +ĠF al +Ġsent ences +ĠJ a +IF ICATION +we apon +ov ation +Ġco at +Ġinter pol +Ġl ips +ĠK y +Ġv ectors +_ am +Ġint ake +.w orld +Ġin box +ĠM AC +_ ab +(name of +Ġent ert +Ġgather ing +ĠS IM +++ . +ny a +' }} +ĠUP DATE +Ġp ac +( html +ĠS ant +i ating +ĠIde as +Ġspr ay +ĠH art +Ġver ification +ades h +/ modules +ĠM ind +ĠSized Box +Ġsh elter +Ġher oes +att y +Ġcert ified +s j +Ġê tre +ÅĤ o +Ġpublish ing +ĠMal ays +.get User +ĠPro vider +ĠLinked List +ĠB or +RO UND +d id +t ain +p ire +ĠJ enn +t el +and e +_f ront +ĠMc G +Test Method +à¸ Ń +Ġoccasion ally +ĠW ales +Ġexerc ises +ĠÐ Ĵ +- plus +Ġvalid ator +Ġpr ayer +L ATED +_ author +Ġlab our +++ Ċ +-e quiv +ĠG PL +Ġface book +s imple +g ly +Process or +ip y +Ġ* > +Ġcle ared +ĠP ush +Ġpen is +Struct ure +li j +ĠM organ +Ġhand ful +" .Ċ +| \ +Ġ ******************************** +ĠA qu +_ IC +.load s +Ġm eter +ĠMar ine +:: { +ĠT S +ĠArray s +.T itle +GR AM +ter min +Ġco inc +El se +_st ates +-r un +m embers +ast ro +Ġon Press +Ġbe ings +Ġabandon ed +Ġtax p +own ers +.m ode +Ġdiagn osis +Ġ_ Ċ +ĠK night +ĉ A +Ġob serve +), ' +! ")Ċ +ĠPar a +Ġvari ation +( False +ĠAnt i +Ġg ri +Ġhome less +? v +Ġbe z +.S erver +re lease +ĠP atri +Ġchar s +Ġrank ing +activ ation +Ġw ides +q r +.S ql +ac ular +ĠB ot +_s ync +Ġhapp iness +Ġvolunte ers +Ġs its +/ < +[ e +(file Name +Ġcap ac +ĠMar ia +f ather +Ġgr am +* i +Ġcas o +_d raw +ĠR aw +ĠIter ator +ĠP adding +P D +BO X +ĠS PECIAL +Ġfe cha +Ġv ide +ĠLe ader +ä» ¥ +$ (". +Ġdiam eter +Ġm ild +Ġrock s +app ings +d irectory +.fl ush +ĠJ ess +UN IT +ĠP ear +Ġmand atory +S ur +q t +Ġstream s +Ġco operation +ĠS ac +Ġche aper +ĉ ch +an imation +f are +( height +( True +N Y +Ġw rest +Ġpoll s +Ġencounter ed +ĠMarket able +_P ASSWORD +_SE LECT +ĠArab ia +_c lock +Ġv oy +Ġи з +Ġst ir +is ible +-e ffect +.c reated +Ġto ys +ĠTrad able +Ġr ust +Ġstr cpy +_t imestamp +Ġtalent ed +, null +ĠJ obs +ĠPort land +Ġweak ness +Th row +ĠAng el +ä¿ ® +Ġun cert +ï¼ī Ċ +ĠìĿ ´ +Wh ich +Ġ[- ]: +S omething +Ġconv icted +k le +ed ium +Ġbranch es +Ġb ases +ç ® +Ġcomplex ity +ĠF ig +. reshape +$ db +_CON ST +ĠT es +.r untime +Ġden y +ĠB SD +Ġk r +h att +ĠSt atic +Ġunivers ities +Re place +Ġdro ve +Ġad oles +_pl ugin +ĠL GBT +Ġt ex +du ction +ED I +ĠT ed +_ URI +Ġre ception +art en +.S ingle +r ice +sc ious +_b g +Ġw ages +ĠS ervlet +UIL ayout +Ġform atted +.M od +< class +is en +Ġrepresent atives +"] = +Ġport al +ĠHun ter +Ġh iring +__ )Ċ +ric ulum +u o +li est +Ġt ears +L at +Ġliter al +.In sert +Ġc urs +ĠCom put +Ġterror ism +Ġswe ep +Ġ[] čĊ +Ġpass enger +Ġeast ern +Ġtwe ets +Ġoper ated +w nd +ĠS yn +.t ools +ĠW M +ul ates +Ġbacter ia +( bytes +.set Data +Ġvis ibility +// ================================================================ +el m +Ġgener ating +Ġm v +Ġk h +j en +/ search +Ġaccount ing +se gment +act ic +. ip +Ġdeploy ment +Ġfoot er +> ',Ċ +Ġexpand ing +ĠHam ilton +ĠCon trib +.T ables +Act iv +H H +ocom merce +_ ; +Ġamong st +ow ing +ĠC old +AP H +Ġpsych ological +_t ensor +Ġpack aging +ĠSw eden +Ġp are +Ġag gregate +Ġmoder ate +_h and +Ġdesign ated +Ġdr um +Ġget User +ĠC reek +_s cope +ĠTrans fer +ĠM arg +Ġfight ers +W nd +ĠS el +ĠLa unch +Ġemerg ing +if rame +ĠAdd itional +Ġf ears +Ġsat ellite +_ : +Ġdis posing +Get Value +Http Post +AT IVE +ul ary +View s +Ġatt ending +ĠT ennessee +ĠM ission +Ġmedic ation +ĠW y +ĠAn na +Ø ¹ +ĠVert ex +.t ypes +O rgan +.DataGridView TextBoxColumn +ĠR S +Ġtemp o +( App +Version UID +.p oint +ĠD utch +H ours +L U +Ġqu oted +.b uilder +ĠPer fect +ĠAl ways +_t wo +Ġexclus ively +ĠC ra +ific ar +ĠA WS +ing ham +com plex +k ernel +Ġgr avity +Ġw i +Ġover view +ĠW ant +ĠW P +( sh +. rotation +St ates +ĠTe en +_com ponents +ì Īĺ +Re ceived +Ġly rics +rit es +ĉĉĉĉĉ Ġ +-A merican +[ num +/ python +ĠU ART +Ġapp le +ĠJon athan +Ġmoment um +ภ± +Ĥ ¹ +Ġm ich +and ra +Ġbi ological +ĠM ens +Ġ% % +else a +ĠMex ican +.rand int +Ġt ale +ĠValid ate +Ġdefe ated +.ht m +Ġcop per += / +cos ystem +Ġr ip +dec imal +.V ISIBLE +ĠT a +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉ +Ġdownload ed +en vironment +Ġnom ine +build ing +ĠSp ot +ipher al +Ġal to +qu et +ĠF T +/ get +/m aster +W IN +åħ ĥ +W est +arg c +Ġprodu cers +ĠM uch +_st orage +cred it +CON T +Ġv et +Ġvo ices +(' ', +Ġinstr uments +ĠM SG +es se +re pository +om ics +Ġdeal er +St ill +Ġb anner +asc ii +Ġrem arks +[ js +Ġshort er +g ulp +Ġmyst er +Ġk un +ĠB ird +Ġti ene +n ut +ĠU m +Ġw ise +Y eah +INE SS +_b egin +- heading +C ourse +Ġ čĊčĊ +omb ie +grad ed +ĠG PS +Ġ że +F it +c aption +ö n +/ image +l ia +(m od +Ġle ak +en za +/ H +ĠH appy +D ist +n x +ĠGovern or +(l ast +te acher +ĠS ent +s upport +ject ory +Ġ Ùħ +Reg istration +ĠGr ay +, false +Ġadjust ed +( settings +< R +ĠM age +Ġpl aint +_ )Ċ +ĉ it +omet ric +. bootstrap +Ġcar ries +I p +Ġ! $ +Ġswim ming +ĠMar io +ĠQuest ions +P ACE +æĸ ¹ +e or +}} " +Ġo ven +ĠK on +Ġwis dom +Ġac quisition +ess ment +ag ine +Ġexpress ions +Sequential Group +F ront +ul pt +aw k +'] )ĊĊ +_ AR +Ġanal og +ul in +_PR INT +ĠL G +Ġb lob +ĠFurther more +_com ponent +ĠC ole +L AN +SCRI PTION +Ġl ap +icens ing +_TIME OUT +ĠF ro +Ġli ability +Ġcom posed +.create SequentialGroup +_p erson +Ġbe am +ĉ ĠĠĠĠĠĠĠĠ +ĠNot Found +. 'Ċ +ÃŃ s +.Text View +P DF +Ġk ar +__ (' +Ġ" :" +_m essages +Ġhar vest +.h istory +> 'Ċ +-f old +æ Ĭ +ĠBet ter +Ġ"\ < +sp acing +Ġfurn ished +os er +] }Ċ +Ġ$ " +p ull +.P ost +( ip +Ĺ ı +.f ront +nt e +ĠF M +g uid +Ġnegot iations +agon al +Ġtrem end +unge on +Ad v +car ousel +ÃŁ e +_DE SC +Ġham mer +áº Ń +ĠĠĠĠĠĠĠĠ ĊĊ +-c ore +-s ervice +Ġcorn ers +ĠS F +p red +> A +ĠJ Label +Ġrom antic +Ġtestim ony +os c +ĠGener ation +as ures +_int ernal +Ġprint s +Ġ] )Ċ +ĠC leveland +re po +D isc +Ġ" >Ċ +�� �� +Ġne arest +_t b +( require +EO F +- child +Ġbu dd +.Xtra Editors +alt ies +\": \" +W ords +Ġloc ally +Ġpurch ases +Draw er +ex tract +Ġexec ut +} '. +user data +Ġfocus es +-min ute +ĠP ublish +og o +Ġmount ains +B ot +} >{ +Ġt ension +ro d +m esh +Ġtransform ed +, R +() }Ċ +.l ong +Ġg orgeous +ĠS chedule +Ġol dest +Ġsub process +( IN +y ect +ĠCo oper +arn ess +ĠMon itor +.p art +ĠN BC +Ġc otton +Ġh ol +Ġrg ba +ĠB io +Cont inue +P od +Ġparticip ating +clus ions +(By Val +à ¬ +ĠH OW +_set opt +Ġaccompany ing +at on +Ġ/ \ +ĠAuth entication +i én +ĠBar ack +/* . +Ġe ager +ĠC ancel +< lemma +ep h +ĉ window +Ġinc idents +), ( +.D es +ib e +ĠFunction s +Ġhosp itals +Ġo xygen +root Scope +Ġd rew +ĉ request +not ice +ak u +am ents +f ar +Ġprec ise +_w rapper +Ġlisten ers +A Z +.b ounds +ĠA verage +field set +_ axis +Ġexam ination +' .Ċ +mon s +++) {čĊ +ĠForm s +íķ ľ +Cpp Method +_tr ace +Ġengine er +ĠFl at +Ġrev ision +Ġhe ating +/ profile +.r u +p riority +Ġin fer +_ST REAM +Ġ* )( +> $ +OLE AN +OK IE +IB ILITY +U AGE +ĠSur vey +Ġres ign +w ing +Ġsecre ts +Ġch ips +JSON Object +Des ktop +_SY MBOL +(res ource +ĠĊ +Ġnew est +ul i +Ġdes ert +Ġd ip +ĠP ow +Ġequ ation +Ġposs ibilities +ĠF ed +os ph +Ġ[ % +Ġb ubble +ether lands +Ġc ement +. auto +_ AN +âĢĻ . +se lection +ĠB ond +D en +- O +.get Type +.W indow +p res +Ġsw inger +" })Ċ +Ġp ip +Ġm ice +Ġcomp ound +- plugin +ik o +Ġcent uries +ic ular +-in line +ĉ key +> \< +EN SION +Ġ[ čĊ +Ġprecis ely +Ġét é +ĠP ast +ĠCam bridge +-f ull +Ġanaly ze +ĠSte ven +Ġn em +d ue +ore n +Ġmus cles +ij ing +/ - +ĠKenn edy +R M +oss ible +Ġact ress +Ġd olor +å½ ķ +Ne ed +.t oggle +ĠR ace +w ers +.m aterial +ĠD ue +ĠP el +# print +Ġindepend ence +ex us +Sh adow +Ġenc oder +( level +ĠSw ift +.d oc +_se lection +Ġserial VersionUID +Label s +Ġperform ances +.T ag +ĠN HL +iz en +/ UIKit +_CONT ROL +Ġearn ings +ĠAl t +_H ANDLE +C tx +Ġpers u +Ġtr an +ç ¨ +_CH ANNEL +Ġsatisf action +ĠG P +io x +m itt +land o +Ġp ig +inal s +ê ncia +S urface +ĠU UID +Ġbenef icial +Ġsequ ences +ĉmem set +Ġmag ical + « +Ġw orn +AS C +pop up +COM P +_b efore +en ess +U i +L es +.re quire +.Serial izable +add Gap +Ġauthor ization +.py plot +urr ay +lat itude +fr ames +aj s +Ġcomp ass +Ġobserv ations +_s up +.en viron +Ġtri ple +ĠRub y +Ġdr ain +_F ILTER +S an +UM P +Null Exception +ĠG ab +ow e +ĠTurk ish +_se quence +ĠGr ant +uel a +Ġw o +Ġc ube +i q +Ġdis orders +Ġextra ordinary +Ġc trl +ĠSe q +ent r +Ġsan ctions +uts ch +Re ports +Ġin herit +Per iod +Ġphot ography +ĠF ramework +Ġspecial ist +Ġ? ĊĊ +_ selected +.P layer +Ġal location +( account +Ġstruct ural +v able +- offset +.App CompatActivity +аР¼ +.Add WithValue +Ġicon s +Ġshut down +_l ow +ĠCom pare +ĠC e += head +l am +.p redict +_DE C +ĠS leep +ĠGr atis +Ġsuggest ion +ĠD EL +ca ff +av irus +No thing +ŀ ĭ +Ġwides pread +Ġmechan isms +Ġtext Align +occ up +ĠR ail +: NS +Ġf iber +Ġm k +Ġv intage +-l ong +.re duce +. Entities +( record +Ġple asant +FR ING +.C ells +OT T +ĉelse if +_con firm +ĠView Group +s ym +Ġpr ay +Ġsus pected +Cont ains +Ġb orders +Ġcomponent Did +ASS ERT +Ġinf inite +- order +Ġh ello +ĠGr ade +.currentTime Millis +apol is +z h +ĉ Object +: \\ +H O +val uation +Ġvoc ab +Ġcou pon +atab ases +.Get Type +L earn +] =" +ĠG ary +ot ive +Ġas h +Ġb ib +XX XX +Ġbal anced +VAL UE +ĠN at +_A d +< E +åĮ º +ĠMethod Info +L IB +Ġconsider able +ĠInd ustry +test s +.set Title +ĠBl uetooth +Ġm apped +ĠBru ce +ĠMain Window +ĉ status +Ġr az +ĠM and +Ġclass ification +Per missions +Ġ---------------------------------------------------------------- ------------ +Ġcontain ers +: set +_x ml +Ġwh ilst +Th rough +Ġval ign +Ġworld s +C ORD +ED IA +ÑĢ ов +Ġsp are +ĠH ad +ĠDE F +(p tr +Ġwarm ing +ठ¾ +Ġcons ensus +ag ne +CT L +Ġì ķ +.M ain +web Element +Ġp ist +Fl ash +App end +.tw img +T ap +Ġveget ables +al g +.s ample +Ġcoach ing +( ind +Cell Value +Check Box +ĠH ell +RO OT +Ġst adium +Ġinvestig ating +) % +st ed +ĠW riting +Ġê ² +Ġun o +Ġ{{ -- +Ġco ords +Ġun ser +organ ization +ĠCr ime +ĠDemocr at +Ġv in +/ file +- api +ĠA y +Ġfund ed +ĠBre xit +ĠG h +ent ina +c ases +Ġd ash +Ġ!! }Ċ +H I +Off ice +Ġcapt ain +Ġwor ship +\ C +Ġglo be +_ board +Ġbab ies +Ġconsec utive +Ġenh anced +ere um +ĠAd vis +Ġgr ain +Ġc raw +ancell ationToken +. alpha +_W ITH +ĠO tt +ĠC ool +.b atch +Ġver ified +(c allback +Ġreg ards +ĠInt Ptr +ouch er +Ġk in +Ġtou ched +it Ãł +ath on +Ġadj acent +Ġaccom panied +LE AR +Ġim plies +Ġh ill +ĠBalt imore +=" - +Fin ally +S am +ic opt +Ġs od +Ġm aj +ĠSh ipping +Ġget All +Ġcoach es +Ġdon ations +il ot +ĠT ar +c err +Ġbad ge +Ġmark ers +ĠR and +ais ed +iss ance +Ġexpl oring +uc ed +ĠIndones ia +Ġbene ath +Ġmagn etic +Ġm useum +match Condition +Ġdis rupt +Ġrem ind +ĠT M +Ġ/ >< +Ġf ool +Ġes k +.N ull +ĠD ies +_OUT PUT +_TYP ED +Ġpaint ed +Ġsoph istic +ĠB ear +* n +_P ACK +Ġdeliver ing +ĠC OUNT +åį ķ +Ġj eg +-c ar +f name +Ġr anging +ĠN eg +/ ******/ +ĠCH AR +Ġul tra +Gr ad += t +Ġjud ges +ĠD ise +ann ers +Ġsc al +_c al +ĠCON NECTION +_ embed +(f n +ĠC raft +ĠP as +") -> +.con vert +.res ource +ĠST ATUS +ô ng +ĠT it +Ġclass room +ĠArch itect +ĠK ings +Ġstead y +/* !Ċ +ĠG ene +) ";Ċ +ic ia +st an +ĠCon struction +um per +w c +ĠC BS +ing ing +-p arty +(d river +M ARK +Ġn ested +ew ard +Ġdepend ency +Ġm ales +ĠO NE +ĠProdu ction +][ $ +ãĥ¼ ãĥ +_LO AD +ĠB ol +el ry +ł éĻ¤ +ĠRe quire +Ġpl acing +xx x +CA LE +Ġth umb +Ch oose +Ġprot otype +VO ID +Ġles bian +Ġtra its +Sh arp +Ġconsum e +Tr uth +Ġaction Performed +ĠEnvironment al +ĠDe an +Ġest ado +s ame +Ġnumer ic +Ġtrans it +. Email +-s ide +_R UN +ĠVill age +_OP EN +è ¦ +.re m +-w arning +any a +Property Changed +Ġ(! _ +( check +il ia +ĠSo ft +st eps +ĠMad rid +Memory Warning +Ġhand lers +Ġexperi encing +Ġins pect +button s +Receive MemoryWarning +chem y +Link s +Ġur llib +.System Colors +ĠE igen +Ġpun ishment +:UI Control +bar a +- set +Ġ}čĊčĊ čĊ +Ġtoler ance +Ġinter faces +. redirect +ighb ors +cs rf +_back ground +. Utils +_H T +ĠInter est +im os +Ġgr ants +Ġexam ined +Ð Ķ +Ġc f +for ge +back s +ĠObject s +_s ent +. entry +ĠTH EN +ell ido +c ia +, res +/std c +. nd +( Int +ĠAuth ors +ĠApp CompatActivity +' { +Ġmed i +M usic +ig m +ce ipt +Ġa uss +Ġtarget ing +ĠKe ys +h n +: ]Ċ +Ġmin eral +à ® +.c a +om ed +Ġshe ets +Ġc amb +Ġdead ly +.in ject +( unit +ĠSe lection +.g ms +( connection +Ġ$ (" +é mon +ĠCurrent ly +pt e +_path s +le af +Ġimp lications +pos al +ä½ į +[ / +anc ia +é Ľ +m ul +c ie +Ġge ile +im als +UI View +Ġs urre +serial ize +IS O +Ġarbit rary +Ġsock addr +.f n +ĠM erc +Ġcast ing +Key Down +Ġnew Value +op ens +T odo +Ġflex ibility +ĉĉĉĉ ĠĠ +V elocity +ú n +row ing +Ġcomput ed +` )Ċ +st atement +Ġr i +_c art +L ow +trans fer +.n av +Ġgr ave +ĠDo or +ĉ alert +.sub scribe +- profile +ĉb ase +ĠâĪ Ĵ +__ ĊĊ +Ġengine ers +Ġexplos ion +Ġd ari +ĉ Log +on al +Ġisol ated +{ i +ĠM sg +F uture +Ġrac ist +-w rap +ĠV ers +b org +IS ION +Ġ ÑĢаР+ĠY an +init With +Ġn omin +( empty +ÃŃ n +ãĤ ¤ +ĉ width +Ġch amber +/ ajax +EM P +Ġnec es +iv os +log ic +*) & +cript s +Row At +ib lings +Ġe ars +Ġcomput ing +Ġm aker +ĠNe ither +b readcrumb +Ġserial ize +ĠWith in +Ġd ell +_TR ACE += a +Ġwish es +-in ch +ĠD or +Ġinnoc ent +ĠD ol +Ġint ens +for ced +ĠB IT +Ġphotograph s +Ġcas a +ĠL en +\F ramework +.S imple +Ġde ar +)/ ( +ip pi +Ġown s +Pl ayers +Ġpropos als +.p i +us alem +D amage +Ġcal ories +ĠCreat ive +Ġ[ $ +Ġ// čĊ +And View +è me +.c ustom +_f actory +command s +_lo ok +Ġstr cmp +Y N +a ired +Ġaud it +о ÑģÑĤ +ĠRe verse +ropri ate +et ics +< vector +.s elenium +. or +Ġpred icate +Ġfinish ing +Ġk le +ĠRep os +ĠK han +ĠM aking +ĠF S +Ġp ute +ĉ state +_S UPPORT +' - +orient ation +Ġexist ed +atur a +Ġexpect s +ĠSh adow +Ġorgan iz +å ŀĭ +Ġsusp ension +Ġu it +Ġsimult aneously +ĠAff ero +: ");Ċ +Ġro cket +c as +eter mine +ace ut +x l +ĠA MD +( graph +ass oci +_C R +.ar ange +(j Label +Ġbe ef +Qu ick +.c ard +] ): +- gr +.G ONE +_C LOSE +ĠNe v +ÃŃ as +Ġste pped +ĠFre edom +ĠW R +NS Array +_r x +_d ialog +Ġhot els +Ġ( \< +ĠD iamond +Ġassum ption +um i +( items +č ččĊ +æ³ ķ +Ġn el +Book s +åİ ¿ +us b +ĠF IN +æ ¬ +Ġcorpor ations +US A +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +.p roperty +ew ise +_ plot +"> ';Ċ +Ġpe pper +Ġsh ed +ĠMed ium +ĠC ookie +Ġoverse as +ed or +asure ment +åŃ ĺ +Ġ' .' +Ġph p +ĠPRO C +Ġexception al +( th +ĠJ et +Ġoccup ied +.set Image +ĠRel ated +uck er +M embers +PR INT +ĠG lo +_V IEW +} ",Ċ +Ġad option +[] )Ċ +ĠMiss ouri +ĠLin coln +eral d +Pop up +Ġf ate +- bootstrap +fe ctions +ĠP oll +_ARG S +in ance +-h ome +. ), +_d one +: ĊĊĊ +Ġdiscuss ing +ĠSQL Exception +Ġelect ro +ĉ req +Ġz w +Ġl ui +Ġover night +$ user +ĠW AY +Ġall erg +Ġdisappoint ed +Ġradi ation +Ġimpress ed +ific ates +Ġto b +CL ASS +Ġc uda +_d et +- post +ul u +Trans lation +-h and +.y ear +ĠM ongo +Ġun clear +. engine +WEB PACK +r ices +_AC CESS +Ġh olidays +per cent +.Id entity +ĠG ov +Ġpassion ate +!! . +ĠGree ce +plus plus +')) ; +G P +Ġexc it +.tab Page +_ cond +Ġspons or +M ODULE +_pro c +Ġ$ Ċ +Ġr ational +.T ool +Ġi hr +cc a +åĵ ģ +ĠE state +IB UTE +Action Performed +ĠS olar +¦ Ĥ +Ġequ ity +t id +Ġrec ip +.s imple +m k +ĠL uke +ĠGuard ian +Ġenc rypted +Ġdomin ant +. place +ĠN V +Ġtong ue +( Get +Ġst ainless +.P lay +Ġe b +ac i +.b uffer +readcr umbs +Ġvacc ine +p rom +Ġuser Info +Ġsl ug +Serial izedName +-w ide +Ġre actions +ĠY ang +ĠAdd s +(user Id +Ġpl ates +ĠM EM +Ġb ail +In side +et ed +Ġels if +Ġs ake +Ġc ycles +Ġì Ĺ +ĉ I +-c ollapse +ĠG MT +De claration +Ġg ros +Ġreach es +Ġcust ody +Unt il +t u +ĠCh en +Ġn x +( addr +ĠO ffer +Ġcol leg +ass ador +Ġm apper +ĠS IGNAL +ĠB loom +ĠH oll +ĠIm per +-d es +_s ite +Pro c +E qu +Ġat omic +ĠW oman +s ent +sc ar +Ġint elligent +ĠGet ting +ĠReg istration +ĠPh ill +Ġkill er +unic ode +Ċ ĉĉĊ +ĠJac ob +ĠCon st +Ġloc ate +Ġca us +ĠSch olar +Ġconstitution al +Ġinfl ation +ĠG ot += array +end um +Ġtransl ated +Ġdiv orce +En tries +Ġs or +ĠQu ote +irl ines +U K +Ġexc el +( opt +ĠAD V +,: , +Ġcontact ed +ĠD A +Ġr ings +ĠIndust rial +.get Context +Ġforg otten +ĠT an +Ġp ants +Ġo v +Ġdec oder +ĠPart ial +Ġv c +Ġbatt les +A rial +FRING EMENT +ir ates +, w +aint enance +ĠO d +ĠTechn ologies +åī į +ĠCar ter +.find All +N ome +B en +ĠUs age +ĠP icture +Ġbad ly +_p anel +Ġpat ent +ĠProt ocol +lot te +ĉ player +je ctions +Ġd ou +_re lease +urn iture +_t ax +ĠF ields +.d ataset +_m aster +CLU DE +ĠPh arm +b st +Ġoper ational +.c ell +Ġident ifying +Ġj wt +t uple +ĠT C +ĠC ro +ix map +- components +gener al +Ġo z +_D e +_d ouble +ĠTo o +.View Group +g ate +d ings +ph otos +Ġgrand e +ol lect +_l in +Ġaw ful +f ilters +Ġaltern ate +es p +Ġcomp ress +e o +ĠS cale +Ġind irect +Ġinv oice +ĊĊĊĊĊĊĊĊ ĊĊĊĊĊĊĊĊ +Start ing +ĠPl ayers +ie le +. then +Or d +ĠT uple +Ġb out +ĠStat istics +Pre view +Ġp uzzle +ĠW idth +ST ATE +Ġover lay +ĉ on +Ġin fr +Ġsm allest +lock ed +ÑĤ о +ss l +Ġde emed +Ġs co +re ck +Ġj Button +Ġmiss ions +ç§ ° +.Selected Index +T ABLE +Se pt +Ġacknow ledge +Ġstrt otime +ĠT ell +ĠD ak +Ġal uminum +Ġf ence +ĠSt ars +CON FIG +Ġretro fit +Ġemph asis +/ header +ĠS omething +in ished +=' ".$ +ĠValid ators +Ġpol ar +section s +.as px +Ġas pir +.M ock +Code Gen +Ġpe ut +Ġaccept ing +Ġback ing +P icture +/ ap +еР³ +_SE C +- use +annot ation +Ġcogn itive +Ġg rip +h our +ĠLeg al +Ġep ic +.t oolStrip +.not ify +.L ast +OR IZ +M iddleware +cri ptions +l ash +_F OUND +ĠLiver pool +Ġ{} ", +Inst all +Ġn it +Ġfig ured +[ len +.W in +.pl atform +Ġgam bling +(d t +av ery +ĉ include +Wh ether +R outing +Ġther ap +Rem ote +ĠL oss +y ll +Ġappro ached +ĠV ehicle +ĠAl pha +Ġvoc ê +ans wers +NS Dictionary +cons ider +un used +ĠF an +or able +f re +ĠDIS CLAIM +ĠAct or +. ] +to Have +.user Id +Ġspeed s +ew ay +Ġrec urs +ĠÐ ³ +_pr iv +! âĢĿĊĊ +Ch oice +Ġsett le +Ġplan es +' }, +T om +IT ER +! "Ċ +å » +achel or +Ġsepar ation +Ġd al +ad j +Ġreg isters +r iz +ĠNot ice +Ġl u +Ġcour age +Ġax es +cell ent +.as ync +Ġcompat ibility +ç « +Ġ! ĊĊ +ĉ title +Y LE +ĉ message +U UID +OLD ER +ĠH H +ĠStyle Sheet +Ġaccess ed +. validation +t asks +Ġpoll ution +.c anvas +Ġing redient +ĠC abin +A h +old own +ĠNO I +ĠÃ Ĺ +[ f +ed uc +y alty +(n ot +_ State +am en +Ġda o +ud ad +ell ers +} & +lic ity +_W INDOW +Ġt atto +val or +.R ange +Ġrefer enced +ĠRes erve +M oney +SCRI PT +/ product +cho ices +Ġt in +ãĤ ĵ +Ġsepar ator +Ġp kg +am med +ĠM AT +! !ĊĊ +Ġr aid +Ġmotiv ation +ĠX P +ĠBack ground +ĠQu aternion +.define Property +ik er +ĉp arent +ĠOrigin ally +ant age +ĠH ans +Ġtim eline +.c ur +op ic +ĠSe qu +m ust +ĠCo al +Ġform atter +_R GB +Ġ_ (" +'} ),Ċ +Ġ= ================ +ĠF UNCTION +Ġl ng +ic ates +l ive +_ engine +Ġtown s +')) ĊĊ +ĠP K +( api +ĉs canf +pack et +.ph one +á Ģ +ĠAnd y +_N AMES +PL Y +Ġmin s +im i +Ġbr ick +Ġbl ade +.std out +}` ;Ċ +Sh ift +ĉs b +ĠCheck s +Ġphenomen on +Av atar +Ġmin istry +ro se +ĉ File +Ġtit led +( LOG +Ġg an +des ign +(), čĊ +Ġb ones +st m +ÅĽ Äĩ +ĠInput Stream +Ġvol unt +ĠSerial izable +Ġfight er +ĠDr ag +T witter +Ġsubs id +ç ¼ +Ġfor ums +.load ing +log ged +_ this +Ġterr ain +Ġir re +ĠIn g +ĠC N +_object s +. uid +Ġconscious ness +T INGS +ĠG all +Ġport ray +ĠDevelop er +Ġparticip ant +Ġ" ;čĊ +/ model +ĠOper ations +^ \ +ĠL ater +Ġrais es +-n one +.m eta +=' .$ +Fin ished +Ġrepl acing +Ġsam pling +ĠJ en +" There +RE AL +A LE +ìĬ ¤ +Or ders +_param eter +ĠOlymp ic +Ġtr ès +Ġare na +i ol +; ?> +Ġimpact s +ĠW S +: get +Ġfl ights +ĠRuss ell +c amera +F n +s igma +Ġfor cing +Ġloc als +Ġdepart ure +Ġcelebr ation +ĠS ay +ï¼ Ĵ +ĠH ills +.has OwnProperty +Ġtyp ings +.A PI +Ġdon ation +Operation Exception +.Act ivity +c plusplus +ĠChar lie +Ġimport ed +Ġd ann +Ġoccas ions +Ġimplement ing +Ġpur ple +.d ialog +SQL Exception +ern o +Ġw ars +Ġpast e +Ġdecre ased +Ġhar sh +Ġel abor +input s +ĠView s +Ġerror Message +_m ul +ĉ write +ĠC op +ĠAnn ual +(b utton +Ġv ida +b ars +ĠHar vard +ĉex pect +Ġindex es +Ġdocument ary +Ġf lesh +OR LD +ĠD elta +M AND +Br ush +-c olumn +Ġdevelop ments +method Visitor +s lice +ĠP DO +Ġinvest ing +ir able +Ġxml ns +ï¼ Ľ +art a +Ġthe ories +_c ity +Ġ$ __ +Cre ating +( pr +D ropdown +ism atch +ĠN ET +'] )){Ċ +ĠVal ues +ĠSE O +ĠST AT +Ġe cosystem +Ġtem pt +Ġ\ \ +Ġ// {Ċ +ĠChrist opher +ĠKent ucky +ĠHttp ServletResponse +Ġhy brid +y on +Ġfeed ing +ĠEx tra +N orm +IT CH +ĠSe an +ĠUp load +m un +p ur +Ġp ersistent +ĠID C +ĠPer form +.m erge +_ room +Mean while +! =' +ĠW el +Args Constructor +.D atabase +Ġcount ing +() * +Ķ åĽŀ +ĠT OP +m ill +ĠD T +IGN ED +ĠK B +Ġcomp ly +S outh +_c ollection +Ch apter +Ġexpl aining +_ AM +_t s +c ards +Ġqu el +Ġp ole +Ġtouch down +ĠO thers +Ġpe ers +ĠType Error +Ġsix th +Ġche er +Ġdis pute +us c +) ], +th umb +Ġh iding +ĠS IG +lik es +ĠP AGE +.Ref lection +Ġhead quarters +T ING +ĠG host +M LE +$ Ċ +Ġcontr ary +ext end +'] ). +FF ECT +ĠP interest +úmer o +ric ane +ĉs ession +Ġcr ystal +- Control +overn ment +og raf +- action +v olume +ft en +Ġun con +Ġan imate +Ġle ase +sc r +Ġref use +ãĢ ĭ +ft p +in formation +Ġeval uated +Ġin jection +Ġj ack +Ġwork shop +æ³ ¨ +PT H +ĠT s +off er +ĉ os +Ġking dom +M issing +Ġlaw makers +ext Field +Ġsing ing +ab i +/ client +.m edia +ATEG ORY +Sign ature +% ',Ċ +ĠF uck +][ : +Ġsens ors +/ com +ĠPr imary +.S QL +_pro gram +Ġp ills +Ġinteg ral +Ġfle et +Ġdro pping +.s l +Be en +Ġp ets +Ġadvis ed +Ġdr agon +_ EDIT +( im +F ER +ĠDr ug +(r andom +Ġcomp ression +ou st +[ % +Ġbuy er +h op +R oles +man age +Ġpain ful +ĠBr anch +-mod al +en ant +ĠM esh +/ font +ĠG raham +Ġâ ĺ +Ġn c +ĠFranc is +Ġspec ification +Ġdam ages +- config +Ġthe oret +sec ure +_m ulti +aceut ical +Ġdemand ing +en ne +IST S +() ));ĊĊ +Re ason +Re cent +ph ase +Ġps y +_M AN +Ġvolunte er +å ¿ +istrib uted +li o +Ġproduct ivity +_com m +S pring +n is +. weight +ĠC ancer +Al loc +ĠT weet +Ġsepar ately +ĉ check +_p roperties +. Unit +_CL K +Ġg t +Ġ( );ĊĊ +Ġhand y +ĠThom pson +Ġunn ecessary +ĠRe ader +G N += request +ĠU tility +.Re pository +ĠA x +hy dr +ie u +Ġth y +Ġl t +_m ail +ä¿® æĶ¹ +ail and +ĠPhil ip +Ġbit ter +Ġbet ting +Ġtim ed +ock s +' a +Ġal gorithms +Ġre interpret +Ġto ss +ro gen +Ġhop ed +( selected +Ġvent ure +TE X +ĠLe ave +.Sub string +Ġgr ateful +uk a +ĠCon sumer +Ġag greg +C ircle +ภģ +_block s +Ġleg ally +Ġ" | +ãĥ ĥ +. board +.A b +Function s +rec ipe +è ĩ +ĠO xford +Ġwho les +.B uild +_ch anged +h ai +Ġdepart ments +I mp +Ġcoal ition +IN FRINGEMENT +Ġemp ower +itch es +N orth +Ġinfl amm +ON SE +Ġmiss ile +ĠR aj +ĠIss ue +Ġat oi +ca led +.Cont rollers +ĠW olf +Ġcrush ers +á» ĩ +.A uth +.add Attribute +h is +Ġbo ots +.c lean +c amp +Ġten ant +Ġt une +Ġ{} '. +Ġwork out +Re po +Ġpartial ly +MI SSION +j amin +ĠS B +Ġdetermin ation +Ġ' ');Ċ +ĠB eng +Ġv os +Ġin hab +/ lang +s burgh +Exec utor +h one +ĠCh allenge +_link s +.Le vel +Ġunder ground +-c ode +Ġoptim ization +log ging +_de st +Ġsn ake +Ġchemical s +_IMPORT ED +ado op +ĠTH AT +man aged +Ġredu ces +ĠRE AL +ĠG uy +_GENER IC +/ ******************************** +. amount +Ġd ere +get Time +Ġp ant +an onymous +Ġharmon y +ĠAl an +Ġscen arios +Ġd irt +ht ags +M c +Sh ell +r in +{ čĊčĊ +.p ow +ĉ client +Ġconspir acy +Ġad mission +ĠReg ional +ĠView Controller +ĠPhilipp ines +Ġde pos +Ġp ap +ĠP ad +P aul +.Com boBox +Ġt utor +ĠRec ipe +w riting +Ġcontrib utor +OT H +Sm all +V I +Ġh acer +e qu +ĠEx amples +h uman +.m essages +ĉt yp +Ġ( čĊ +ĠS SL +LE N +ĠRom ney +( grid +ĉ min +Ġ> ĊĊ +Ġfr uits +Ġvot er +In line +pan e +ĠC ollections +char set +Ġsp am +z b +item ap +Ġsucceed ed +_C OL +Ġel apsed +im eter +Ġrecover ed +T ensor +hatt an +.set up +ist o +( head +ĠS IZE +Ġtact ics +Ġdist ur +Ġpre val +ici os +( Value +_c ols +ĠF at +Ġse al +Ġs ons +Ġens ures +Ġpress ing += & +igen ous +Ġharass ment +_ JSON +Ġign or +yn omial +om er +_st atic +Ġsignific ance +Ġcirc les +_S ystem +Ġdiscipl ine +Ġdress ed +Ġs phere +Ġclim b +_ actions +ĠB ab +Ġ' =', +_s chema +" use +Ġund ers +Ġc ups +.s creen +/ new +Ġappe aring +T OP +vis ed +cl ang +Ġinvestig ators +Ġmyster ious +Ġprom ising +Ġqual ify +Ġc ave +Ġequ ip += x +G T +( link +. velocity +. erase +ot er +++++ ++++ +pro fit +Ġz ones +_ uid +- ser +Ġobject ives +Ġmil f +web kit +(m atch +ne h +ĠAssoci ated +ĠT odo += d +C am +Ġv ocal +Ġs udo +( EX +Ġtr ou +AB C +.b ean +ĠG round +ĠRE ST +we ets +In g +im on +_b us +ĠC OLOR +un to +Ġf oss +ĠLink s +ä ng +/ forms +pr ises +Ġachie vement +C ALL +ел ÑĮ +ĠVer ify +_S OURCE +apt cha +ID D +_re ference +G old +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĊ +Re ceiver +Ġa j +_d irection +} ] +ĠCom pet +Ġb ang +ĠC ass +- url +te chn +ĠJer usalem +long itude +' );čĊčĊ +Ġwin ners +T asks +ĠD MA +Ġtool tip +İ · +ĠB ra +_d uration +cur y +parent s +---- >( +ĠK ir +Ġint ros +Ġsk etch +Ġsk illed +Ġim mer +Ġade quate +_re p +( header +_ like +Ġper ceived +ss h +Ġassum ing +Ġf f +_u uid +ul as +Ġdemocr atic +. entities +S eries +aph ore +Ġnew er +} ( +SE C +ai ro +Ġcomm od +Ġprivile ge +Ġde ux +ĠH op +.' / +ct ic +. ';Ċ + C +ĠWar ren +Ġoptim izer +ĠSER VICES +_ oper +get Attribute +ĠMc K +_s elf +.r s +" )ĊĊĊ +Get Component +er ce +Ġt ous +un its +'] );čĊ +Z oom +/ E +Ġobs c +Ġfast est +on line +Ġpeace ful +ff en +Ġc argo +ĉ pr +Ġseek s +z u +Tr im +Ġw ard +Ġver d +Ġblog s +.exception s +ĠPrem ium +ĠN etherlands +S afe +Fin ish +ĠAl bum +_A CC += this +v irtual +] > +_L ABEL +ĠN ich +_w in +ĠA aron +W P +; $ +aim s +ĠImage View +Ġend less +ER A +_DIS ABLE +Ġcancel led +- us +Ġins pection +em in +ĠG rey +- open +Ġiter ations +. owner +Ġk eras +.P assword +ĠR y +ĠIN S +A ir +ĠSe veral +.Tab Stop +ING LE +ĠH air +ĠCan vas +AA AA +Ġfl aw +ced es +.Re port +í Ĭ +ĠT ips +cript ors +.trans action +.S pring +Ġview er +Ġins ights +è¾ ĵ +ord ion +U INT +se ek +ĠA uf +ìŀ IJ +Ġstr ain +To oltip +Ġd z +ign al +ad t +Ġu c +fin ite +Ġn m +.c md +ĠMy Sql +[ data +.j ackson +.t ree +Request Param +_ agent +") ]čĊ +Ġass ass +( Constants +: ss +ĠM AN ++- +- +ĠB ottom +print s +ĠS ame +@ Autowired +sw ap +ici ón +Ġprotest ers +Ġh oney +ĠV eter +(C alendar +- ad +ĠBrook lyn +L ife +_V AR +ze ch +ĠC ALL +_C AST +ĠE lection +Ġthick ness +V ery +_IN TEGER +- dev +)) )) +ap at +oo oo +d emo +Ġparse Float +ĠR ather +ST IT +m aker +[ current +chron o +Ġch rist +ãģ ª +ĠD etail +Æ° á» +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġs ul +id ency +Q ue +Ġeleg ant +ap ons +Ġdish es +Ġinteg ers +( read +find ViewById +ĠAm ount +ĠSk ip +Ġhab its +* )( +Ġmon sters +M AC +: end +Ġfr ank +As sembly +Ġd fs +Ġne ut +_TYP ES +e qual +loy d +( uri +Ġch i +Ġdefend ant +Ġconflic ts +Ġv il +- js +ĠPe ace +Ġmut able +) sender +ĠF ocus +å» º +Ġapprec iated +s leep +ĠR ED +C ulture +Ġdesign ers +_g enerator +c odes +/ ex +.Get Value +umb led +.scal ajs +per or +Ġveter ans +Ġ} )čĊ +Ġun fortunately +_C REATE +M ass +ĠCL AIM +ĠMe et +_s upport +B ank +() .Ċ +D ark +_LO W +ĠMin ing +ĠO wner +ier a +Client e +Ġencour aging +> S +Ġboy friend +ĠH alf +ĠA CC +A ff +_ ar +-l ife +c x +.J Button +iz ado +.z ero +.open qa +ot on +.text Content +Ġto ll +at ie +Ġball ot +- number +. Exception +ĉ params +c ircle +-m ap +Ġn ap +ĠRob ot +ĠI ch +reg istration +Am azon +roll ment +( exp +Ġt anks +ĠG ordon +Ġmach inery +Ġbas eline +æ ĭ +Ø © +ĠCon vention +ĉ config +ook ies +m ult +Rec ords +ĠE ST +Ġgar bage +Ġcon form +id al +Ġb arg +Ġsurv ived +Ġinvestig ations +.contains Key +---------------------------------------------------------------- ----------Ċ +ort ion +Ġhor r +_ http +Ġm ant +] ;čĊčĊ +b inary +em pl +Ġin quiry +ĠMean while +Ġcollect ing +.Entity Framework +", ĊĊ +ĠP ic +@ Inject +ick ness +ĠB inding +Ġcont rolling +re verse +Ġch airs +semb led +( add +Dis abled +an as +.trans late +-------- ---Ċ +Ġref lected +"] ĊĊ +Ex ternal +Ar row +Single ton +% x +Ġ Å +Ġan cest +ĠOr leans +ĉc md +Ġprohib ited +ith metic +(ch annel +_c ss +For ward +.s ocket +Ġl uc +â Ĩ +ĠFire fox +ĠM ovies +) _ +. ends +( shape +Ġde alt +Ġs aves +Ġgl ory +Ġmej or +Ġbreath ing +Ġ eller +get Data +Ġang les +Ġtool bar +Ġsp acing +IP S +Ġflo ors +_ACT IVE +Ġsh uffle +/ shared +ĠE le +ed ish +Ġweb cam +.ex pect +il oc +ĠIn cludes +Ġtweet ed +Ġ: ) +ĠEss ay +F ix +-b etween +_ web +.con v +Ġrac ism +Ġreflect s +um m +иÑĤ е +_f ooter +/d ocs +ĠP our +Ng Module +.initial ize +pattern s +_ In +ĠAb b +* čĊ +Ġsent iment +b uff +_count s +Ġre use +ch unk +Ġim posed +Primary Key +Fore ground +Ġconsum ed +? ! +Ġd ick +Ġch ron +ĠF ern +Ġrespons ive +Ġin sect +icult y +Ġr w +Ġal ike +Ġsub set +ĠCook ies +ĠP air +Ġt ier +IF O +av our +ĠQ U +, sizeof +Ġmerg ed +m v +it ol +yl on +Ġjump ed +. role +ens aje +R ules +Ġb rowse +An imator +Ġy oga +Ġvari ants +Ġcour tesy +ur an +p bs +else if +Al t +ĠL ane +CL K +IM ARY +_PRO PERTY +ï¼ IJ +Ġch an +Ġgrad ually +Ġsh ake +Ġbl onde +... ");Ċ +-se x +Ġgame play +ac ies +.ref resh +US B +ĠPl ot +W as +iss ippi +ĠT ensor +Ġcryptoc urrency +Ġdifficult ies +De leted +With out +_ append +_ ver +")) čĊ +Ġhonest ly +Ġp ivot +Ġtem ps +_p s +ĠUn like +[: - +V S +_in f +Ġjun ior +Ġanim ations +Ġfile path +? {{ $ +Ġun icode +pl aces +ĠC offee +.S E +ĠP AR +(t xt +ge bra +Ġf ires +Main Window +med ium +Ġ( âĢľ +Ġl g +Ġc mp +/ base +_l ayers +_ entries +Ġadmin ister +ĠSU CH +B P +ĠScott ish +ĉčĊ ĉčĊ +gu ard +ĠStr ong +In sn +ĠC AP +as ury +ĠSE E +C lock +er ie +\ models +Ġ$ $ +ĠC ab +Ġwur de +Ġsold ier +Ġcl ips +Ġarrang ement +ĠW onder +ĠH orn +Ġsc ared +Ġc ure +m kdir +Ġal igned +ĠP ink +Ġland ed +Dim ension +Scroll Pane +.ch at +.W ith +ĠTr ain +] .Ċ +Ġth irty +Ġdur able +Ġl d +Ġlate init +Ġch arts +Ġins ult +.F atal +_ ct +Ġm asks +CLU DED +Pres ident +Ġcol ours +g ments +.at tributes +ĠF lex +ĠC lock +ÃŃ cul +im en +J O +ĠReg ex +_L INK +Ġc ouch +ĠIN PUT +Ġbe ating +b usiness +pre ced +. unit +ĠF el +N ever +osp el +.start swith +ĠE PA +. only +Ġprevent ing +y er +Column Name +Ġelev ation +fl u +icy cle +Ġoff line +Tool bar +Ġcompet ing +) ]. +Ġm og +Ġis Valid +As k +_ av +_l at +AN C +ĠJ oh +k ers +Ġgu ards +Ġch ains +ĠSimple DateFormat +.st atic +Ġvess el +Ġm ud +Ġst abil +Ġst ret +g m +am ation +ç ľ +-w ith +Ġro s +_P A +Ġresult ado +Ġconf idential +ĠTok yo +ĉ using +ĠMath f +omb ine +ĠESP N +Ġdeal ers +Ġdismiss ed +TR Y +Ġte ens +rec ords +Ġw ings +g allery +account s +_L IB +Ġj acket +ĠNS Object +Ġst ones +ĠDel ivery +ĠD iet +/w atch +Ġto ilet +ĠG uest +.d ay +Ġint val +Vis it +Ġinvestig ated +Ġpent ru +ĠThe atre +andid ates +L ang +ĠS erv +Ġcont rollers +Ġset Title +N P +am y +fl at +( ui +_d ocument +è ĥ½ +ĠC oin +ĠAd ams +pt ic +Ġproduct ive +Ġaccompl ished +čĊčĊ čĊčĊ +Ġdefer red +ient es +Ġs inc +ol ars +Right arrow +Ġvari ations +( offset +.Layout Inflater +Ġsus pend +Ġprevent ion +_pr ivate +_ js +âĺ ħ +Ġw ieder +at um +Ĵ Į +Ġappear ances +.D ocument +Ġvalid ates +cal endar +} ";Ċ +.d emo +con ut +Ġcorre ction +ĠDe al +Ġbatter ies +.d uration +, \ +_m arker +m ulti +Ġh alt +Ġc ms +Ġsh aped +B ro +re duce +Ġ #### +CT OR +ĠBen ef +Ġicon ic +Ġp iano +Ġeffect iveness +| .Ċ +Ġa jax +Ġv olumes +ภ¡ +Ġcl js +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ath s +ra its +å¤ § +Ñ ĸ +_m ult +Ġfasc inating +A verage +Ġpr é +ĠChair man +.find Element +_p in +Ġcomp aring +Ġdark ness +-F i +- server +Ġselect ing +ster dam +ĠPart s +FORM ATION +Ġnot ing +Ġp ile +og s +Ġpa lette +_d o +it ize +() ( +Ġdef ining +Ġremain der +Un its +_T ASK +Http Client +S ocial +Ġfund ra +N R +ch est +C urrency +.ad apter +Ġd op +un ting +ANG UAGE +" He +ĉ index +_p ackage +.I con +Ġrep et +m ass +=" .$ +ĠS ud +Ġl id +pro vince +ì ľ +G PIO +Ð ļ +ĠMy SQL +Ġdoc s +ĠG A +Ġip sum +K ernel +Ġaccept s +Ġfit ting +Ġcu ando +Ġd uplic +ĠBro ther +ĠK le +num s +Ġmor ph +Ġ ######## +ĠCG Point +< unsigned +ä¾ ĭ +ĠD uke +.set Bounds +q s +or ic +j er +Ġregard ed +Http Request +Ġbond s +Ġthorough ly +enc ent +Ġhighlight ed +Ġac res +Ġwork place +ĠL ux +Ġqu ot +.in flate +Ġdocument ed +Ġadd iction +Ġmut ation +.c ity +Ġbott les +ĠRepos itory +on n +err no +ARI ABLE +åº ¦ +_B EGIN +gl as +' })Ċ +ĠMass age +ĠWh it +reg ex +W A +Ġout let +- head +Ġexp ired +ĠTh ai +/ include +grad ient +scan f +Ġse am +w al +ĉb uf +B earer +Ġprec ious +if acts +co ord +Ġexpl oration +.get Y +(h andle +Top ic +ĠV ent +r hs +---- --Ċ +ĠB right +Ġg uild +m other +st orm +Ġmunicip al +Ġin k +.T YPE +w l +... manual +ĠTechn ical +Ġcorpor ation +ĠH W +ank a +T AIL +ist as +Ġperform s +ĠBeh avior +.F or +_ ORDER +ĠK ick +Ġcallback s +_d r +ue go +h ub +uff icient +sk y +Ġb p +ht able +ĠON LY +ĠAUTH ORS +.Arg ument +" };Ċ +ĠTh under +ĠK om +.Sh ould +A UTH +ah u +_p ayment +Ġst arter +ìĦ ľ +ìļ © +B log +.p atch +Ġgovern ed +ass y +-f ound +Ġthe ater +ĠFont Weight +ĠBat man +" If +.R andom +_d elta +ĠC E +Auth enticated +Ġdr one +Ġc ous +r adius +M er +( None +ĠN J +_ headers +Ġam er +py test +ĠA ctions +ĉĉĉ ĠĠĠĠ +Ġet t +Ġh oly +Ġun comfort +ĠN in +ĠDec imal +ĠM essages +.s ender +] ])Ċ +Ġembr ace +Th ough +/ sp +Ġcult ures +Ġhigh way +t ar +.f ail +_h idden +ĠcomponentDid Mount +ĠW right +Ġj ag +_ il +../../ ../ +ig u +F ood +Ġa ce +Ġa ños +US D +Ġmut ual +Log ic +Ġtem ple +Ġbrief ly +ĠT rip +class method +default s +Ġch unks +,, ,, +ĠRe ason +$ id +-up s +Ġdam n +Ġtruck s +Ġun limited +Ġsc ulpt +ĠC ards +Ġaut or +ĠTest ing +Ġdies e +sh ops +ç ´ +(p ayload +ĠP ATH +ĠMem orial +Ġridic ulous +eg ree +-w inning +Ġre hab +Ġsophistic ated +wp db +ĉ path +! ";Ċ +_S YS +.s peed +Ġso ap +s uffix +W rap +Ġenh ancement +à ī +ú b +Ġplay list +Ġmix ing +ant idad +=" ";Ċ +ĠRev ision +ĠBe at +.in c +-w ay +enc ias +ul ers +C at +id el +ĠSh ip +.set Color +Ġthreat ening +.mod ules +Ġafter wards +ĠD ashboard +Ċ ĠĊ +Sign al +Ġpr imer +orne ys +ici ary +Ġl igne +_p redict +Ġa est +_ https +> : +ĠL ex +Ġrencont res +eg ral +sc ala +_f amily +ÃŁ en +_s ym +Ġuncert ainty +ĠVAL UE +Ġ} ;čĊčĊ +Ġbro ader +Ġh orses +ãģ Ŀ +ĠK al +ob a +_IN ET +ĠK ill +j query +am ination +[ @" +Ġm uj +## #Ċ +First OrDefault +then Return +C he +/ footer +Ġpark s +as je +ĠG ulf +Ġmod est +. Init +ï¼Ł ĊĊ +Ġpros pects +Ġs vg +Ġå ı +.D ialog +_N ET +Ġ( ($ +Ġe k +ĠW arning +ĠM K +< LM +Ġ' čĊ +i em +h etic +Ġi x +th ink +-sh adow +ĠE ld +ĠNev ada +ĠLe af +ĠG ROUP +Ġprom o +ent ine +ĉ Map +ĠModel s +ĠK rist +_k ernel +-m ade +Ġc err +As sets +ell ar +Ġinv oked +.v ue +Ġcult iv +C losed +Ġgener ates +ffff ff +thes ize +s qrt +ĠCast le +.c ar +Ġke en +und a +ĠC row +ĠSing h +y thon +Ġbe ans +l arg +æĸĩ 件 +Aw esome +unc ate +Path s +o ji +(c urr +CON DS +Ġm im +Ġshould ers +H ard +ast es +а еÑĤ +Ġconv ince +de cess +m ade +ĠC MD +. Im +Ġcha os +ens ively +Ġcool ing +Ġbur ied +(' @ +_S e +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉ +.com pany +.sub mit +ph ant +Ġboot strap +_h elp +à § +.d ump +Ġdif er +_m apping +Ġcirc ular +Ġescort s +Ġb ere +Ġgrad u +ĠLeg end +im edia +ĠBar celona +Ġbed s +åĪ ° +ãĢ Ĭ +_v olume +Ġtremend ous +Ġsc aling +Ġp ins +en as +type param +D ashboard +render er +Ġsp i +Ġ& $ +ĠSk in +alm art +Ġh ockey +Ġ'" .$ +Ġerr no +Ġb ew +Follow ing +.M odule +er able +ĠM ilitary +ĠR io +_ available +ĠSur face +Ġst ab +IF IER +ĠL IST +Ġd ashboard +Ġcl usters +.pl ugin +Ġj ou +ĠDec or +F our +Ġdel le +****** /Ċ +ia z +in de +ch ing +Ġget Item +.Add ress +ment ed +A meric +Pl ain +Ġus b +ĠPract ice +_ ment +.bl ue +H int +ÑĢаР² +Ġconn ector +Ġinher ited +и в +Ġinterval s +Ġc ere +Ġu d +Ġin con +.Ex ists +ĠM ic +F K +(c ard +.Set tings +Ġexhib ition +Ġon Pressed +Ġrest ored +eng u +. def +Ġrec v +." );čĊ +enc oder +ather ine +( dest +az ed +# endregion +sem bl +, M +ob y +Ġп еÑĢ +.C all +Ġattend ance +-b order +Ġaddress ing +ê n +ĠLe v +Ġb ash +ben ch +C redentials +Sp acing +( of +_RE SET +ig uous +Ġcr uel +Ġcross ed +Ġle ur +ĠG olf +or rect +Ġpack ets +ĠData Set +Ġpart ly +SEQU ENTIAL +Ġindic ation +ĠS alt +ac ia +Ġ* );Ċ +ĉ info +ĠView Bag +on z +Ġeditor ial +ĠA rena +Ġs ir +_ Static +( socket +s u +cho ose +.m onth +.M y +é ri +; font +do es +Ġcon verter +Ġsal v +Ġl r +Ġinflu enced +(f eature +ĠQue ens +let t +_M ON +& amp +Touch ableOpacity +O FF +Ġmetab ol +( iter +Ġvit amin +ĠIND IRECT +aut om +_p ublic +Ġadjust ment +Ġspecial ized +w indows +.add All +Ġaccording ly +ĠJ OptionPane +Ġcell spacing +Ġqu ad +Ġcre ep +Ġout lets +}` )Ċ +Ġpri est +_TH READ +ĠMar x +ĠBy Val +Ġc ual +éĿ ¢ +Ġtempor arily +An n +ke leton +å ¥ +ĠLO C +au er +der ive +Ġbeh aviors +as ename +ĠCent ury +Ġhor rible +ME SS +_ List +we i +P at +ĠCh oice +_F ROM +ĉ line +.in voke +.B ottom +Ġnow here +." ĊĊĊĊ +_ export +Ġstrugg led +.Ap pearance +ĠJ Button +ĠJer emy +([ [ +Ġkick ed +mar shal +st aff +es ity +Ġqu iz +_e ffect +Ġ} ));ĊĊ +m el +b anner +ĠP IN +Ġin vention +Ġcons olid +Ġop s +ĠB etween +j ack +ern ational +Ġsacr ifice +ag ation +ĠJ oy +Ġam endment +ĠS old +Ġprison ers +ан нÑĭ +Doc uments +) ])Ċ +ust ed +ĠLine arLayout +os o +_E M +.s elf +.M iddle +) // +Ġ\ ' +Ġfuck ed +ĠM urray +Ġprof ound +_E LEMENT +ult a +il ers +port folio +J une +t cp +mod ified +ĠTr ace +ĠK el +aly zer +) => +ĠRep air +_B E +Br and +u art +pre view +Ġiniti atives +run ning +b ang +ĉ update +ĠCo ach +R ich +Ġy outube +Ġrit ual +app a +ĠRobin son +prec ision +//////////////////////////////////////////////////////////////// //////////// +=[ ]Ċ +Ġcelebr ated +OT O +Ġin clusion +J P +' ;čĊčĊ +Ġnot able +(_ . +Man aged +Ġgu ides +& nbsp +ated Route +ĠAd just +Ġcol ored +_s cores +ĠTes la +_pro gress +.in st +[' _ +.fl ags +Ġf close +_O PER +ż y +_n ote +Ġtrans gender +å ķ +RI PT +Ġabs ent +Ġam et +Ġoper and +ë © +Ġh ood +to LowerCase +av o +ĠCirc uit +ĠL ind +-- }}Ċ += m +Ġsup press +ĠM AP +i ang +- admin +Ġside bar +ĠB u +ĠH ex +, F +ĠSign al +Ġtrans parency +ĠFeder ation +/ V +Re q +Ġpul se +Ġt ends +Num bers +% ' +Ġde port +dat as +_U INT +_ tra +ok o +Ġ" ? +comp et +sole te +und ry +Ġover lap +}` ,Ċ +. ly +_sum mary +ĠL ost +.C enter +Ġdis ability +.Serial ization +Ġge om +Ġ? : +ĠW o +Ġsh ipped +Ĥ æķ° +Ġu gly +Ġexcit ement +Ġext erior +Ġcheck out +Ġk ur +, D +ĠAl aska +Ġsyn thetic +ĠB udget +ĠSub scribe +Ġ& Ċ +ÈĻ i +ĠY u +ĉ query +} .Ċ +Ġtr aged +ass en +Ġaccommod ation +Ġphys ician +Ġren amed +Ġtid ak +z Äħ +Ġmin us +ny ch +_EX CEPTION +thread s +Ġt ire +_c reated +ens ure +Ġworth y +Ġexc use +Ġclo th +.parent Node +/pl atform +ĠU FC +ĠG tk +un ny +Ġg ibt +ke ley +h um +(t x +ĉ dev +Ġout fit +do ors +Ġf on +ic ut +vol atile +Ġhom osex +Max imum +Ġexp end +Ġ});ĊĊ Ċ +E q +ond ers +dep artment +ĠPhys ics +" });Ċ +Ġpar ad +.S tr +Ġse le +IF IED +Ġdel ivers +iv an +Ġrespons ibilities +Ġadvoc ates +è µ +ĠR ID +.param eters +M etrics +ron ics +ĠUITableView Cell +A bsolute +ip se +yl um +MLE lement +_VAL ID +< title +D lg +p aces +Ġsynd rome +be ans +_d atabase +oz illa +ĠM eg +DB G +Ġl ub +Bag Constraints +ab ad +Ġproject ed +_BY TE +.Size F +st reet +ĊĊĊĊ ĊĊĊĊĊĊ +ĠLO SS +Ġdirect ors +/ news +Ġnurs ing +ĠD one +. HTTP +dis count +ĠR ot +To Many +Ġen abling +Ġauss i +ost a +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +è½ ½ +Ġhel icopt +ĠIn side +ä¿¡ æģ¯ +is per +ĠAll ah +ARCH AR +Ġroll s +Com pare +X P +Index Of +S UM +Ġass ured +ĠPhys ical +End point +.G lobal +.d etail +Ġthe ft +.j upiter +Ġhum or +.R ender +A lex +.c ap +Ġbuff ers +Ġdis pose +t ion +.p resent +z el +, P +Ġdesper ate +.get Column +Ġtw in +ì ĸ +.c an +Ġf lee +ĠIran ian +Ġstick y +ĠU TC +L T +//////////////////////////////// //////////////// +Ġl icensing +_PO INT +ĠM aps +Ġl ol += models +-t ab +ĠN ash +_log ger +tor ch +ĠCON SEQUENTIAL +Not Empty +/ react +Ġp f +Ġassert ion +Ġsubsequ ently +_c an +Ġpand emic +og ue +"+ Ċ +_ ent +_P aram +.ĊĊ ĊĊĊĊĊĊ +Res earch +C apture +Ġbel oved +d em +Ġextract ed +Ġf ights +ER C +(a uth +position s +Ġrevers ed +(st ack +Ġ_ ) +uto ff +_fl ow +ç Ĥ¹ +( Game +Ġex cluded +ĠCS V +c g +ĠT itan +p ause +Ġcer ca +Ġdump ster +L ess +Ġkotlin x +aster xml +Ġpoint ers +Ġfl ows +ĠT un +ĠMain Activity +Ġdis cret +Ġcomb inations +vis it +_b ind +oot ing +d ater +_look up +.n io +Ġswe at +ĠR d +Ġscient ist +ĠP ixel +@ NgModule +Play ing +Ġunf old +Trans late +ĠLaw rence +ĠFIX ME +B ill +ĠR IGHT +Ġwhere ver +Ġo ok +vid ence +Ġ] ]; +ĠSk ill +unist d +ĠðŁ ĻĤ +Ġfem ales +-- )Ċ +İ· åıĸ +ĠF red +Over all +Ù Ĥ +Ġess ence +Ġthere by +Ġw ounded +ĠD OWN +les son +text ure +R ound +Ġautom ated +ĠÐ ¡ +ĠUp dates +Ġsh ade +p ublish +ĠG ear += lambda +Ġle ver +) +" +h ill +Ġrad ar +ry ing +Ġ" ). +f illed +Ġline up +Ġd l +Ġworks pace +V o +_d t +ë ² +_ Item +NS URL +. verify +ĠHawai i +G od +M arch +Ġ[âĢ¦ ] +Ġpel o +ur ious +ĠPitt sburgh +. It +C lean +> \<^ +Ġi os +s ound +"] ; +Ġfre ed +rot tle +ĠL ower +[ count +å Ŀ +Ġp ale +ĠWay ne +ear th +_c ategories +U CK +.m etadata +Ġsum mon +H OME +олÑĮ з +Ġmanufact ured +Ġdo ck +Ġcompet itors +_MODE L +ok ia +ĠH ey +Î ¿ +Ġback ward +ĠPO SS +rop a +Ġc ri +_O BJ +Trans port +-h igh +Ġerot ik +_s lot +Ġart ic +_f ramework +-ser if +ĠSql DbType +') ( ++ "/ +Ġw ore +S il +Ġst oring +ĠPh ase +u ant +Ġb ump +in ho +Ġd ign +Ġback s +q q +(h ash +Ġge o +Ġt ender +Log o +! )Ċ +ĠM X +ĠAr thur +esso a +_C h +Ġbed rooms +="# ">< +Ġth roat +ins ic +.int eger +Ġpr imitive +Truth y +Ġfacilit ate +Ġcreat ivity +ĠD NS +Ġg ra +ue z +Ġcount less +ĠPol and +' M +ĠD ist +Ġv est +Ġcert ification +á» ij +h eld +ext ensions +( static +Ġgr ades +ĠU ber +ãģ Ł +Ġ[ ])Ċ +dat os +Ġget Data +ĠCh arg +ĠB S +.m icrosoft +.v ideo +.d irection +->{ ' +l ua +ape st +Ġbo iler +ere k +Ġdec ides +.j ar +IS C +ĠW ords +(C ON +EMPL ATE +ree ze +sh ots +app s +unt ed +.set Name +:: < +-b old +ê ² +å¯ Ĩ +Long rightarrow +Ġunf air +Ġear ning +Ġsh elf +URE MENT +Ġid le +_M ENU +.C ustom +AG ER +- " +_s witch +b ecause +) view +m are +_ condition +ĠStart ing +M vc +(p re +d ump +_LO CK +at etime +.c allback +ĠC er +op ol +ib rary +Ġres ervation +ĉĉĉĉĉĉĉ Ċ +lect or +grad uate +Ġgener ous +Ġ ion +ric ao +m q +_com plete +(c ursor +ĠForm Control +: center +Ġsub stitute +ĠPl anning +Ġp ension +Ġrecommend ation +ĠT ags +Ġg ef +Ġalbum s +Ġwash ing +ro c +Ġtr ains +at ings +Ġex ponent +ack bar +- ln +á g +.Data Annotations +ĠE IF +ĠMalays ia +ĉ PORT +on us +Ġcle ver +Ġpe u +> ĊĊĊĊ +ĠArg uments +Ġdebug ging +( right +' D +com pute +Ġfin est +OR AGE +Ġspect acular +ph rase +Ġind ia +Ġlegend ary +b irth +Ġcom posite +Ġg rows +ĠT D +Ġep id +Ġlaunch ing +] ][ +Min utes +ĠCh a +Ġclean ed +Ġwitness es +uk an +ĉ Type +Ġhab e +par agraph +ĠJ Panel +ĠH ann +Ġvar ied +ĠP okemon +ĠM UST +åĬ ¨ +.vis ibility +op up +^ [ +.exp and +Ġ" ', +.f asterxml +_ auto +ĠShe et +mark er +Par cel +ew s +ĠStr ategy +-m aking +Ġun ve +Ġtrail ing +Ġclick s +ĠGet Component +ĉ content +IG ENCE +ERN EL +NSMutable Array +Ġb reat +Ġharm ful +¶ Ī +Ġbes ides +Ġb oring +Ġbrut al +v ang +(p arse +qu ick +Ġpy test +Ġswitch ing +() ]Ċ +Ġì Ħ +L ER +ĉf ont +Ġnet t +) ]ĊĊ +(/ \ +æŀ ľ +to Array +Ġbre ed +ĠC AR +ĠWe apon +A bs +t ot +Ġset Name +apt ive +Ġ: , +Ġesc aped +ord en +ĠP ri +th umbnail +Ġdescri ptions +/ styles +ĠPC I +Ġal phabet +astic search +NOT E +Ġc ialis +ĠGr iff +Ġpor que +Ġprote ins +pl ays +Ġst ating +Ġimag ination +Ġfac ial +ĠMe chan +Ġarr anged +_ used +Ġarrang ements +ĠP ipe +host name +Ġprov inc +T it +.Flat Style +ĠS plit +ĠLo ader +.c c +Ġclin ic +---------------- ------------ +Ġb aking +ĠEN T +ne ath +ãĢģ ĊĊ +AN E +.EntityFramework Core +app ers +. ic +ĠNg Module +ĠF ORM +Ġ' ; +-pro fit +h w +en emy +ĠE ye +Ġca ution +t own +Ġur ged +ĠJim my +ynchron ous +-s ized +m aking +, { +] ', +_ Object +ah oma +Ġactiv ist +IN VAL +ĠCom mercial +ĠOr lando +(t ab +ĠØ ¨ +Al gorithm +Ġher itage +Get Mapping +Ġfail ures +ri os +at iva +Ġt et +Ġcar pet +( Z +th ree +Ġdisc losure +. ERROR +_c alled +Ġd ial +Ġoccas ional +.E rr +Ġfunc ion +caff old +Ġrele asing +ï¼ī ĊĊ +_ Value +ĠV ari +y ellow +Ġstrugg les +.c al +ĠDak ota +ĉc lose +Ġsand wich +Ġanaly tics +Ġ** ) +& # +ĠJ os +Ġpass ive +AT TR +Th rowable +ĠM un +ĠU int +(dis posing +ar ak +ĠLe aders +Ġaffect ing +Ġitem View +Ġeconom ics +f v +๠Ģ +.r b +ĠOver all +Ġwealth y +Ġev olved +nd a +ĠH us +re strict +um en +ĠA gricult +! ĊĊĊ +Ġexp ires +Ġspokes person +int erval +Ġà ¢ +Ġque en +(n il +ing o +He ap +Ù İ +Ġcompl ain +S ym +ĠCl one +ĠR u +ĠW ILL +ĠCr ystal +/ content +ing en +oint ment +Last Name +av icon +ĠIB M +ĠDim ension +an h +icip ants +ĠAn ne +.pro gress +Ġal go +ob il +ĠV oice +ĠF E +Ġg li +Ġv ed +Ġprevent s +\ Column +Ġfol k +ett i +Ġm n +ĠCL ASS +Ġdisplay ing +ĠK l +ĠF err +d uto +. ib +Ġd ados +' name +-s pace +Ġit alian +Ġin verse +Ġd ense +ut er +ĠI Enumerator +-s ign +Ġnation wide +Ġperson a +Ġsol ved +Ġdram atically +Log out +Ġgr av +Ġanalys es +ol lo +Ġl amp +. team +ĠE rot += [" +Ġd ancing +Ġ?> / +Ġc ater +ff e +ĠSh a +ĠB os +ĠRE QUIRE +ĠMon ster +ĠR B +ĠI DE +Ġsu its +Ġform Data +( theta +Ġsp atial += NULL +ĠSql Connection +Ġ à +ĠV enez +ĠMor ning +Ġpublic ations +ĠNON INFRINGEMENT +first Name +ud s +W ould +_HE AD +Ġinvest ed +st able +f red +Ġcommand er +SE S +âĢĶ a +an che +ĠM ovement +ë ³ +S uite +Ġjur isdiction +ë¦ ¬ +ĠB eth +j Query +ĠIs a +Ġd ental +, * +ĠL imit +ili ation +=" { +b ast +Ġt urb +is y +O OK +Ġadvoc ate +im ag +LE CTION +л ÑĮ +(c ategory +.de c +Ġun iqu +_s n +Ġattract ed +Ġà ī +ĠRun ning +_ edges +ĠDis able +_A S +åĽ ¾ +Ġnetwork ing +_br anch +H aving +toBe Truthy +G I +Ġcamp s +se p +-p art +Ġ)ĊĊ ĊĊĊĊĊĊ +ustral ia +ĠRe ports +rit o +Ġwa ist +_pl us +ĠW W +-p erson +Apr il +Ġs ar +.t ar +Ġagricult ural +t ic +Ġt cp +Ġset Value +agent o +ĠAp pe +p iler +CA DE +Ġan che +atch er +Ġcom ics +Ġl bs +_se gment +'] =$ +itt ers +ich er +G INE +Ġutil ize +ĠC ursor +_ex pression +Ġd ag +< long +Ġr hyth +æı IJ +Ġconsult ation +Y et +")) ĊĊ +_M AC +c ould +Ġ' \\ +ĠV o +ĉ http +Ġg s +ph er +- grid +J ames +J ul +Ġsch on +Ġtensor flow +ĠLOG GER +am as +Ġsc ipy +Ġconv iction +. ag +Ġadministr ator +)) {čĊ +Ġn un +" group +P or +Ġnur se +ex pression +ak y +ĠHe avy +. opt +.get All +Ġover l +/ ", +_c ountry +ç İ +ĠG ENER +_r oute +ĠD al + ´ +ol oad +Ġuncomfort able +(m enu +Ġhost name +' ");Ċ +Ġcalcul ations +-c lick +Ġprotect ive +ãĤ ¯ +_F orm +ung s +Act ual +m f +ĠProcess ing +ĠIn ventory +(m atrix +app ropriate +w eg +ij a +Ġch r +Ġr ifle +-w sj +k ar +Ġindepend ently +I OS +Ġconsist ency +v n +/s ystem +ĠCh anges +Ġexp ose +ici ents +Ġrel ate +ĉ next +è ¨ +ud es +Ġglass es +F XML +.... .. +ĠP df +Ġappro ve +Ġ{ \ +Ġexist e +)) ( +ARE NT +оР¿ +ĠL atest +ĠNiger ia +.Inter faces +Ġrem oves +En emy +Ġen force +vert s +ĉ pos +_text ure +W ARD +ĠINC IDENT +( container +Ġdef ending +ĠR X +ĠH ook +br is +ĠFl ask +Gr ay +. )Ċ +vis ibility +ĠRedirectTo Action +err al +_e lem +Ġres on +front end +_variable s +ater ia +Ġ+ " +ave led +RI X +Ġdef icit +_C heck +YY YY +To One +sp y +Ġun ited +end ent +Ġp ode +ãģ Į +C AT +(f mt +ĠBon us +Ġre ck + º +Mod ules +Ġvac uum +R adio +ĠDAM AGE +P en +ĠPark er +; ;Ċ +ĠRe ally +_n eg +p ending +Ġnomine e +ĠC ategories +ĠUl tra +We apon +Ġdef ender +I ss +ĠG ender +ĠD ress +Ġimpr ison +Ġbank rupt +imension al +PH A +ĠStr ateg +ĠPROF ITS +Ġp atri +//////////////////////////////////////////////////////////////// //////////////// +de legate +Ġfor State +Ġdev oted +_m ake +Ġterror ists +ĠS nap +_n av +ĠA A +ĠI an +ĉ app +Pl acement +_h dr +< K +Ġs ang +st roke +- Q +> x +.T ask +m oney +ib aba +' });Ċ +ĠSpec ific +ĠLine ar +_O PT +Hash Code +( Player +.Contains Key +Ġcoll apsed +trans parent +_R ANGE +View er +(c fg +Ġsort ing +Ġinf ected +ĠN ach +Ġaccommod ate +.element s +_P ART +ĠSex y += get +( year +Ġx hr +: ] +ows ki +Ġsum mar +Ġ ¿ +Ġint e +Ġwork flow +ĠTai wan +vers ions +åı ij +Ġsurprising ly +Ġopt ical +Ġpro ces +Ġdisag ree +Ġnue vo +ĠC AM +sort ed +le ases +ist le +Id ent +ĉ event +ject ed +Ch unk +V ars +.pro vider +Ġproceed ings +Ġin clusive +Ġart work +end ants +ï¼ļ Ċ +se en +Ġl ig +Ġm akers +_f un +Ġlength s +Path Variable +[ item +ภµ +De ad +FFFF FF +ĠUr ban +up les +ich en +(null ptr +.s pec +, System +UR ATION +(j ob +å¼ ı +Ġtrack er +Å Ļ +ĠM R +ĠSQL ite +Ġd to +Ġ; ;Ċ +Ġm int +ĠInt roduction +ca o +Ġquestion ed +Ġf itted +rev ision +s q +Ġm ig +_un its +_ async +Ġf lick +});ĊĊ Ċ +Ġnot re +}` , +F ilters +Ġm undo +_d ays +Ġfr m +ut c +Ġval s +ew idth +ĠGener ator +ĠArt ist +ĠID s +ĠArt icles +re ater +ĠComponent Fixture +. = +Ġr ou +- no +.b ukkit +eg g +ĠD iff +atic s +Ñĥ Ñĩ +âĢĶ ĊĊ +ĠChar lotte +by e +Ġ} );čĊčĊ +ĠV ik +ĠB row +Ġl v +ĠG ib +-w ing +GL IGENCE +(I l +ĠEngine er +.W ait +ĠP ictures +Ġr het +Ġth ermal +Ġpr aise +< >();ĊĊ +ĠSp ider +P ause +ĠB aker +Ġsl ower +Ġ} ]Ċ +_en queue +Ġdisappe ared +ĠT icket +IN UX +_LOC AL +аÑģ Ñģ +@Inject able +comm unity +Gesture Recognizer +åĽ ½ +Ġsca les +Ġ- ( +/ '+ +ĠS it +Ġexecut ives +ard ing +Ġad vers +Ġback wards +ĉ context +ĠH amp +ĠP F +ĠDe ck +ĠCra ig +A merican +Ġb ell +Ġpro l +uf en +Ġr ng +ar shal +ĠSim ply +first name +sh ore +J uly +Ġmort ality +ĠâĨĴ ĊĊ +Help ers +Ġbench mark +em ade +Ġorganis ations +.g son +ĠText Field +Ġciv ilians +.Array s +ĠMiss issippi +Ġinter mediate +get User +_cl uster +Rel ative +fore ign +.querySelector All +Fore ignKey +Ġreason ably +-------- -Ċ +C ards +ĠK am +ĠTh or +Ġroll er +-e lement +ĠC urrency +dd ie +ALL Y +ĠR A +Ġper met +aa aa +Ġhom ework +ĠV it +Ġm old +ĠF er +[ start +Ġstatist ical +Ġsc ary +_H OME +.B egin +Con struct +ogen ic +ĠDEAL INGS +Ġtamb ién +ix on +. ind +ac re +Ġtransform s +ĠN ap +.B lock +uss ia +pir ation +ul ent +Ġce il +Cl ause +na ire +T ES +Ġne at +ST D +ĠReg Exp +per form +: ) +Ġun ions +Ġs ublic +Ġw inds +lo ating +g lich +Ġp agination +S kill +App ly +ĠOper ator +ist ogram +Ġqual ities +C ross +Ġde com +], " +ĠJ uan +.mod al +.Ch ild +ĠRog er +STIT UTE +:CGRect Make +a lette +Ġst a +as ide +Ġbl ur +ĠW a +if etime +re ed +control s +Ġb ins +Ġп ол +*/ ,Ċ +U IS +ĠR ou +ĠDem o +- awesome +ĠCh ain +Ġh asta +ĠB art +. KEY +Ġvend ors +nof ollow +ĠD est +_b uilder +Ġarg ues +_ answer +g oto +ĠRES ULT +ĠM ON +Ġp oder +o ons +_C ASE +Ġrep lic +Ġfin ancing +ĠD ATE +c ern +_tr ack +t ies +/ logo +ĠNE GLIGENCE +get Type +> T +b et +g irl +ĠINCIDENT AL +-s ite +.tr igger +ĠL isa +_input s +Ġrel atives +Logged In +Config ure +I K +. accept +Res ume +ĠD raft +Ġ* >( +ĠW A +ed ian +ern ess +ĠLayout Inflater +*/ čĊčĊ +oth y +Ġoblig ation +Sub scribe +Ġth umbnail +ex ist +Ġins isted +ĠU ICollectionView +ĠAng ular +Ġtable ts +ĠImp act +ãĢį ĊĊ +ah o +Ġcharacter istic +g d +Ġ= ================================================ +our t +` . +App ro +Co ordinate +Rem ember +Ġmar ine +] ==' +ĠAdmin istrator +.get Default +Ġforg ot +ĠStruct ure +V ue +ars ing +m oment +k w +_c ursor +Att ack +Ġath letic +Ġdiagn osed +Ġend e +åĪ łéĻ¤ +H ouse +ĠP ARAM +Ġw iki +ĠO pp +Ġcons ervation +Ġs nd +_t em +sub str +ĠC ape +.s im +UT ION +an an +âĢĻ un +Ġg y +- work +Ġcomp elling +=' # +ĉs ub +Ġdirect ories +íĬ ¸ +Ġtouch es +out ines +.C ollection +s chedule +.l at +ĠDo ctrine +CA A +ĠRe fer +Ġshift s +Ġlik elihood +pre ter +ĠF emale +Ġinter cept +Ġl ou +çĻ » +Ġr ug +ĠC rown +Ġ************************************************************************ **** +- product +Ġprompt ed +ung le +d ocker +ĠT u +ĠUn ique +_ Error +ul os +Ġâ Ħ +Ġ( ` +Get ting +_s cal +ĠEn h +ü t +Ġsust ained +Ġp atches +Ġpros per +ĠG aza +_l ight +Ġin cons +-------- Ċ +ĉĉ ĠĠĠĠĠĠ +S F +C N +: ";Ċ +ĠColl ins +( *) +Ġcomp ilation +'] čĊ +Ġcon sequence +, ... +Ġd m +ĠB LOCK +Cl uster +Ġsk i +(arg c +T uple +Ġjo ins +ĠSher iff +W ar +ind i +Ġcomment ed +H OST +Ġinv itation +apan ese +Ġperm its +preced ented +_z one +ĠA my +_R D +Min imum +Ġinv ocation +.en able +icht en +- owned +" id +_PO INTER +F ac +Ġspecific ations +Ġnom ination +Ġg p +< ( +Ġrob ots +ĠJ erry +Ġhold ers +Ġw and +c ms +Ġ} ))Ċ +.To ast +ĠI List +B ased +z oom +/ style +ĠBe ck +M en +Ġcontrib uting +Ġund o +ĠO H +Ġadd Object +Ġe igen +sign up +éĶ Ļ +Ġdist ant +PAR ATOR +ĠM ari +Ġm á +E mp +ó s +Ġì Īĺ +ev t ++ j +p ark +ĠSt ay +ĠD un +Ġso y +> % +az ines +Ġti empo +(m e +p resent +.Th is +Ġedit ors +F IELD +.W ork +ĠUn iverse +Ġdr unk +.t imer +Ġalter ed +ĠN ar +ëł ¥ +.Act ive +id or +ç Ń +.delta Time +Ġawk ward +& quot +ĠSaf ari +Ġtr icks +MENT S +div ision +Ġvary ing +ĠHigh way +Ġphotograph er +ĠSt ewart +Ġlast ing +.P re +.amazon aws +ĠL uck +.D escription +ĠN az +n eg +Ġc ó +<<" \ +ĠSur v +ĠU nc +Rec ipe +.Border Style +Ġmod ifications +- at +AT FORM +h dr +ak o +Ġsublic ense +ĠJ ump +Ġbe im +ĠMan hattan +. bool +_h w +ÑĤ ÑĮ +B in +Ġg ateway +" ": +ĠU IS +:" + +- def +ĠReg ular +/ testing +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +string stream +Ġdis par +Ġmob il +- read +ĠAd apter +ĠCh ampions +Ġsched uler +Ġk ills +ĠM ultiple +ir ror +Ġgod s +AD O +ak te +ĠUs uario +.c ircular +Ġre cept +ĠEx pr +Ġelder ly +Ġnic ely +Ġbest e +W ant +Ġclass ical +.s prite +obj c +ĠM ason +Ġsist ema +.Bl ack +es o +ĠZe it +Ġdiv id +Ġent ers +_sub ject +ĠPlan et +.w arning +ĠG ram +_t okens +Ġhousehold s +_c ustomer +user Name +c ross +Ġp ione +Ġass ists +_S M +ib o +Ġlo yal +Ġuse less +# elif +ĠUlt imate +C ome +g el +Ġd ich +xy z +ik el +ob ra +_s can +ĠInter ior +ĠN ice +Ġpl ac +ĉt arget +Ġvir al +ass o +() / +und e +ĠAd obe +O s +vis ited +ĠO W +ĠFe ed +ĠSe quence +Ġman ages +in son +ĠLouis iana +{ }) +ĠH ab +ĠL D +Ġb ip +pr ites +(e lem +.h ibernate +él é +Ġoh ne +_trans action +Ġann unci +P ublished +ĠH onda +ĠT am +ĠP acket +_ selector +Ġchalleng ed +Process ing +-h over +Ġtr ainer +_c ancel +ĠNS Dictionary +ab ric +ĠM LS +_s ensor +Ġshr ink +ĠF X +th reshold +ĉH X +-m ark +` .` +S cheme +(f ull +_w riter +ĠS ys +Ġf led +ĠC in +-w idget +ĠPre vious +G ender +_ question +Fe ed +Ġscr ut +(p refix +ãĢĤ ãĢĤ +Ġin fections +Part s +Ġhier archy +_DE LETE +ĠPat ient +_p ay +Ġprom oted +Ġì ĭ +Ġcivil ian +Ġagricult ure +ĠP iece +Ġst ance +uts che +Ass ign +.A CTION +F ig +_r adius +ĠS ync +du cer +f ailure +ens ed +pt ime +B M +_dat etime +qu ivo +QUE UE +èĢ ħ +Ap pear +Ġsum mit +: void +Ġv ine +è® ¤ +on ne +_TR ANS +.g reen +_ cc +Ġhung ry +Ġ" > +() );čĊčĊ +Ex tract +iz ens +Ġsol ver +Not ify +Ġeng lish +ĠSh opping +inter faces +RE Q +Ġil leg +ĠUI ImageView +Ġdis connect +ĠUnt il +ĠConserv ative +@ Column +Ġshift ed +Ġ: čĊ +Ġf ich +Ġd la +Ġsh oe +"), čĊ +ular ity +_RE SP +We ather +UI Application +. iterator +Ġag ing +.P arent +ow ie +(e qual +ĠCon v +/ default +Ġmeas uring +.pre v +.Is Valid +.F at +Ġs Äĥ +key words +with out +Ġso vere +Ġex changes +Ġm elt +Ġis lands +ĠInt egr +Ġjump ing +Ġg le +Ġjournal ism +Ġd ated +Local ized +ĠRef resh +Part icle +Ġa a +ĠSTR ICT +Ġb od +.Pro cess +_A UTO +ĠP ublished +e very +Ġtechn ological +ls x +Ġir rit +Add itional +Ġdel imiter +_l anguage +- area +bo ys +ĠT ube +Ġw at +Ġmechan ics +_ owner +Sp ell +ĠSt ories +.Append Line +Table View +h em +st ick +oll ower +I FF +ĠU V +oll ision +S UB +Ġcompar able +Ġdon de +s ales +ll vm +Ġ} ],Ċ +OTT OM +ĠPur pose +L ab +Ġinterview ed +o is +as il +.set Id +ĠIn struction +-- > +ĠMod ified +ation ally +ĠMe eting +è¯ ¯ +# region +Ġrout ing +.f ocus +ĠYou th +< D +ĠN ag +contact s +Ġform ing +Ġm ie +',[' ../ +ĠB P +Ġapp et +ĠTe acher +ĠT P +Ġann ually +outed EventArgs +ĠSpe aker +Ġre name +CF G +(" // +æİ ¥ +/p ages +Ġpr és +ĠSp ell +.All ow +ĠINT ERRU +Ġ( # +âĢĻ ĊĊ +_G eneric +.im show +_t im +- face +(& ( +atin um +Ġrevolution ary +ĠH ours +r ain +Ġany time +Ġab b +.j sp +Scroll View +ĠTr uth +Ġanticip ated +Ġacc ent +. checked +Ġspec ifies +Ġca f +Ġcell padding +Ġcook ed +ĠH ugh +pe ek +_R ATE +Ġd orm +/ čĊ +IV ITY +.Cont roller +(p art +.con straint +Ġinv asion +MO VE +Ġgl uc +l ename +Ġam en +eng lish +ĠSw itzerland +";ĊĊ Ċ +pe st +.col lect +N ib +ĠD ict +ĠE mb +(sub ject +Ġoutr age +Ġdec iding +Ġsent enced +F echa +" A +Ġqu er +Ġfont Family +Ġqu adr +- Y +_C ACHE +Ġanaly zed +Ġg aining +ĠAgain st +ĠSou l +ta u +Ġlight weight +ĠT F +ĠEffect s +.T ypes +.add Class +Ġv egan +é ģ +.' " +ĠExpl orer +.d etect +.sh ift +Ġoblig ations +last Name +Ġassoci ations +ĠTime Span +un ter +ĠF resh +Compat ible +P ub +id ges +. option +var i +.hash Code +Ġg eb +. section +- not +ĠSub mit +T N +reg istry +_m edia +Ġn aj +ff t +Ġm ate +-th ird +Ġp ockets +est a +Ġb ent +ĠN ord +Ġretail ers +ĠMor ris +."" "ĊĊ +W rong +Ġ ÅĽ +R ay +. ec +ĠB ind +_H AND +(n on +is Valid +Ġsimilar ly +_L IMIT +Ġdynam ics +Ġdist inction +ãģ Ĩ +< N +Ġor th +ĠToy ota +ĠK ate +ĠL S +or ie +ĠSpr ings +Ġf reak +last name +_M ULT +-st ep +" ( +AD DR +Ġentert aining +_CON F +Ġdec oded +Ġst reak +Ġwait ed +Ġnot ified +rodu ced +vis ual +.Layout Params +æ ° +es ian +f its +s pring +ĠBern ie +User Defaults +Ġped est +Ap pearance +ĠW iki +ĠNOT ICE +Ġs sh +Ġdur ante +ĠZ ip +ı r +ĠNAT O +Ġtw elve +Ġro yal +ï ¸ +Ġmer chant +ĠF urniture +'] ),Ċ +, X +Ġfold ers +ĠG ate +ĉf unc +p ick +_us uario +ĠV erm +ment ion +ur pose +Ġalert s +x ious +_s ig +ĠF u +Ġ( : +Ġd umb +åħ ³ +Ġaccur ately +éĩ į +R B +-s creen +ĠV ER +j our +Ġrom ance +uc ceed +. choice +Ġad ip +_d ims +Serial izable +ãĤ ĭ +.j ob +Ġpro g +uch ar +Ġg ently +ĠR SS +ict ured +_ENABLE D +ĉ label +aw ks +ĠEn sure +rem ember +ìł ķ +Ġtrans mit +{{ $ +.Trans action +ur se +_rel ative +Ġs ized +ĠX X +ĠPr incess +ĠL arry +Ġpr ó +ĠÑģÑĤ ÑĢ +Ġs isters +estr uct +Ġcheck point +: length +ĠCar los +/ icon +_T ARGET +T okens +Ġpat ience +ĠSe lected +q ty +.show Message +Ġwild life +ĠP rops +b m +- arrow +Ġpar cel +fire base +ĠBen jamin +cess o +.t im +ĠG arc +. any +ĠHOW EVER +ĠK o +Ġgrab bed +_f rames +Ġobject AtIndex +ĠADV ISED +Ġsub ur +ĉ GL +Ġ}) }Ċ +-l ength +ìĭ ľ +ĠPot ter +_b uff +.g ui +ĠEnc oding +E lect +-m essage +Ġ � +Ġ ÈĻi +ĠArgument NullException +а ÑĨи +Ġmin imize +Ġrespond ing +$_ [' +ĠInd ividual +á c +ĠIN TER +Ġmast urb +ĠB in +(' $ +ëĵ ľ +Ġopen ly +Ġ> < +Ġun to +olog ically +ĠM ul +VID IA +Ġsl im +ĠCommission er +( on +Ġunder neath +/ db +v ote +( Message +ĠP ope +Def ined +Ġsw ift +ur f +Ġadapt ed +SE L +Ġreven ues +Ġdiv ine += y +Grad ient +_ act +Ġ/*! < +Ġpoly gon +ĠF DA +ĠC arr +at ables +(std out +Ġrefr iger +Ġco ordin +avor ites +ÑĪ и +Ġcompass ion +ĠPOSS IBILITY +- secondary +ur acy +Ġcomp romise +_A V +_ os +Ġbes ide +ĥ Ŀ +Ġl n +.pl ugins +Cap acity +al ah +.b in +ĠC RC +_b alance +Ġflex Direction +Ġam bit +Ġnick name +ĠFor ces +C LE +ĠSh ell +Ġs ail +ĠW riter +ĠA lice +d w +ĠInd ians +ĠMar shall +_S RC +Ġnormal ized +ĠJ ag +ãĤ Ĵ +ze it +r pc +ÃŃ c +.in line +Ġtrav ers +_n umeric +Ġutil ities +Ġev ac +IN PUT +ĉ register +M X +ĠCamp bell +Ġdatas ets +Ġdem anded +Ġinitial State +g an +Ġe i +Un expected +- web +tr ait +, Y +ĠT odd +Ġske leton +Ġoptim ize +ç¬ ¬ +ĠU pon +ĠSt Object +Ġap lic +.' P +v ron +. UN +Ġpaint er +izar re +Ġl av +Ġp om +p reg += function +( serial +ific a +um ing +åľ ° +ãģ Ĥ +- op +U CH +ĠH end +.prop Types +Ġy o +Ġrout ines +Ġcar ing +S em +Ġres erves +Ġprior ities +red its +IST R +Content Type +ĠSch w +/ media +Ġe str +Ġclim bing +- week +cher che +s ensor +To Array +ĠMont real +Ġcloud s +ĠInject able +ĠR ice +Ġpropag anda +_pro vider +Ġind oor +Ġin aug +Ġdipl om +Ġmess aging +_m ut +å ¦Ĥ +Ġk w +ON S +ari ans +R PC +) ]čĊ +-r ay +ĠS or +m all +Ġmarket place +Ġv tk +M a +og an +ig i +Ġspons ored +ĠD ani +.S EVER +>' .$ +m ultipart +ĠW ol +Ġtable Name +ĠUser name +Background Color +Ġf right +_E MAIL +Sept ember +_val s +op ia +Ġsp otted +- Ch +Ġdata Source +/ "Ċ +ек ÑĤ +ĠRequest Method +ĠRe place +-d o +ah n +ĠPh D +] .ĊĊ +N ON +g ement +ĠTh r +Ġquiet ly +Ġtort ure +Ġte as +ĠC Y +Ġa tr +develop ment +-d etail +Ġlight er +Ġarg uing +Ġdes erves +Ġcur riculum +_CON TEXT +ÅĤ y +H ITE +ĉ ID +/ uploads +Ġt its +re o +_d rop +. UTF +Ġpick up +Ġgro cery +ĠP ure +Ġeas iest +Ph il +.f eature +(" * +Ġinvest or +t ok +Ġj ar +L os +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +. queue +-s peed +M al +um blr +ĠCON ST +ĠH RESULT +ĠD ance +(file Path +Ġattrib uted +ॠį +ĠB und +co ins +Ġs ão +Ġp ir +person al +Ġpre lim +Ġprop ose +ĠT L +] ]) +ĠSub scription +ĠK re +, len +.First OrDefault +) -- +_product s +.Get Bytes +Sh ip +Ġenc rypt +ĠS G +ĠM yst +h ir +Ġiter ate +Ġint end +.mock ito +Ġch apters +( angle +ĠV lad +è® ¾ +' .ĊĊ +Response Body +ĠAb d +de al +Ġbar riers +-out line +b ill +ĠF alls +_se cond +. include +. ceil +Ġoccup ation +ph ony +.move To +ĠJenn ifer +AST ER +; ">< +ĠEn abled +Ġtermin ate +ĠI o +l ations +ĠTHE ORY +Ġear liest +Ġr ack +ĠSc ar +sh ake +ch ip +Ġu v +Ġall iance +п иÑģ +ĠGOOD S +z ione +ĠV I +Ġ{ - +Ġfilter ing +Ġmis con +.Dock Style +Ġb ush +Ġj unk +æ Į +ĠQ UE +Ġhook s +Ġfirm ware +Ġmiddle ware +d ic +ĠOak land +Ġarr ives +P ayload +p ixel +] | +Ġstart Date +.P RO +_a udio +Ġmid field +igid body +ĠSw iss +ĠCl ip +ĠD ump +ĠText Box +Ġg eh +y ield +od s +Ġrefer endum +Back end +ĠC ream +Ġdomin ated +ĠArch ive +Ġrid ers +.prepare Statement +Ġqu ando +Ġche f +w iki +in el +am pling +(" \\ +Ġs ag +_pro xy +ãģ ķ +p do +.getElementsBy TagName +Ġdemonstr ation +ĠN PC +Ġarch ivo +end ance +Ġefficient ly +( actual +.t ableView +Ġm ush +Ġbe ars +_thread s +j as +ah un +Ġne ural +Ġdesign ing +ĠG DP +Ġlift ed +çĽ ® +ĠJ oint +ĠIn clude +ĠGi ants +Ġwithdraw al +ĠR ent +n ative +ĠSe ek +gress ion +_C PU +\ S +ĠSh ield +Ġsol ic +Ġbo om +yect o +Ġmanufact ure +ĠâĢ ĭ +Ġb box +Ġearth qu +ollect ors +:@" % +Ġlo ops +J e +alk ing +ĠWh ats +ĠBo ys +. book +ARG E +_p ixel +Ġsus pects +Î ¹ +us p +ĠBM W +ie ces +(p erson +å¼ Ģ +é » +ĠPod cast +Ġb ou +( Item +à » +( Input +Http Get +Ġb urg +) ^ +BO ARD +*/ , +Ġg ulp +ĠB enn +Ġdeck s +.status Code +Ġac ute +Ġh ug +ug u +Ġp led +," % +h ape +Ġз ап +ĠMain e +.re al +Ġd alam +ĠMin or +.F loat +dis p +Ġt l +Ġen count +=> $ +Ġf g +te es +ĠRec omm +ä l +Ġchem istry +Block s +O ID +Ġfore x +ĠApp end +Ġ{ * +ĠSup ply +CG Float +(b l +Ġat e +ador a +Ġg ust +Ass oci +> .Ċ +F ETCH +.s erial +widget s +ard less +ie fs +_F ULL +ernet es +ĠP red +Ø Ń +äº ĭ +ub ernetes +ĠL aura +Ġl abeled +High light +Ġanno ying +/ update +(d escription +Ġintim id +$ c +")) )Ċ +.A P +Ġ[] * +ĠEX IT +.H ost +ĠOP EN +.send Message +_c amera +_t ile +Ġth erm +onom ous +Ġdis adv +Ġna ar +index Of +ĠP P +.prot ocol +AF E +Ġtext ures +################################ ################ +umb ai +.st ats +ĠG E +Ġi e +ĠST D +ĠM ann +.ref lect +K B +Ġd ive +.w av +/* ---------------------------------------------------------------- +/ settings +.l ifecycle +Ġda ughters +or us +ub er +N ING +st ri +ĠT ip +Ġz n +Ġswitch ed +in et +uff y +ĠTransport ation +( conf +fr ica +ĠX L +ĠLe ad +_per cent +< Map +Ġthr ust +or b +ik k +Ġtra uma +Access or +ĠF it +ĠString Buffer +ex pl +(s creen +Ġaud iences +ĠO PTION +_ round +[ node +be h +-> __ +per missions +ĠD etermine +.M an +Ġadv ances +. InputStream +Ġstrong est +Ġe Bay +Ġ# - +Ġdir name +ĠS MS +Ġmedic ations +Ġam ended +Ġchurch es +ĠImper ial +$ row +ĠMad ison +ĠIn sp +Ġaff air +Ġpsych ology +v h +Ġsever ity +âĢ IJ +Ġstri ps +A H +vert ising +Ġcon se +IM AGE +ĠSt ats +ĉs c +.C ursor +Ġfree ze +ss on +(x ml +ĠSus an +.t ile +ed ed +ĠĠĠĠ ĉĉĉ +uel le +ĠMitch ell +b ased +Oper and +½ æķ° +ĠF F +ĉstr cpy +ounc es +ild o +.execute Query +Ġapproach ing +ĠSe ven +Ġn uts +Ġr ic +ass ignment +Ġcalcul ator +ĠMur phy +ĠB ou +í Ħ +Ġbut t +Ġt icks +Project s +il ib +.text Color +m ov +_log o +( template +ĠIN IT +Ġimage View +scri ptions +OR ITY +Con sumer +Ġun precedented +Ġtour ist +Ġbr on +Ġcontract or +Ġlic ence +ĠN am +æ ¯ +( transform +_AT T +P ref +ĠG am +Ġvess els +Ġh av +L ater +.To Lower +Ġurl s +Ġbreak down +Ġpen alties +Ġf oster +ĠU E +Ġcl ue +com ed +åIJį 称 +-m ain +Ġp ts +Ġcount ed +ict s +/ post +Ġget attr +Ġp ing +ANCE L +Ġp ec +Ñħ од +ant om +ĠBlue print +ĠEvent Emitter +Ġl ä +æ ² +Ġstr aw +( comp +' une +> N +- client +es Module +-b ase +Ġret reat +_s imple +ĉĉĉĉĉĉ Ġ +fe e +') čĊčĊ +Control Item +Ġsubscri bers +ple ase +ĠE ff +Ġp ound +ĠBy tes +ĠTe a +_ activity +Ġmax im +Ġop code +B SD +. constant +; } +omb res +Ġcare ers +) .ĊĊĊĊ +Ġsp reading +-exp anded +ĠOr d +amar in +Ġmob ility +Un fortunately +ak k +N L +_ redirect +ĠP G +ĠS ensor +b ol +t ap +_MEM ORY +ĠUI Alert +plit ude +We bsite +ĠLog o +lo ve +[ ind +Ġalto gether +Ġwonder ed +Ġes per +ĠLib eral +Ġo ss +Ġel it +Ġst iff +od ox +_ment ions +ĠDou glas +_p id +ĠC K +ĠinitWith Frame +.b log +p kg +ang hai +QUI RED +u u +Ġm kdir +AT AL +Ġun h +in ces +st h +Ġhypo thesis +Ġc ata +ĠT B +ĠCl ar +Ġpre decess +Ġsitu ated +-w orld +)) / +Ġhead lines +.st at +Ġout break +sp ath +_FLAG S +ĠServlet Exception +S un +F ROM +ĠD ir +ãĥ»ãĥ» ãĥ» +_co ord +ĠOpt im +Mon itor +.b it +XX X +Ġtod as +f eld +ÑĢ и +im ir +Ġpolit ically +Ġmolec ular +Ġtrad ed +Ġ{{ $ +ĠSw edish +Ġ'@ / +_RE AL +Ġw arehouse +t oday +, L +or p +< section +- br +ym e +ĠUser Service +Ġlib erty +Ġmoment o +( Image +< size +S ch +Ġj og +i ology +arent ly +Ġquant um +ĠAb u +Ġr im +Ġman a +Font Size +Build ing +st airs +AIL ABLE +Ġ& ' +Ġs ect +Ġs igh +(b atch +.I Container +p oll +ĠCor ps +Î µ +ar u +ĠK ay +.r ange +_click ed +ĠRobert s +.N etwork +fin ish +- Man +Ġcolleg es +ĠF ine +")) ,Ċ +f ilm +Ġrem inded +Ġgest ure +out il +Ġthread ing +Ġobj et +Ġt ours +activ ated +.m kdir += user +Ġre de +f ü +_SY STEM +p v +Ġcon gr +Ġmass asje +Ġpract ition +Un iversity +Ġtab index +Ð ĺ +S ets +Ġcount ies +g uest +f an +Ġword en +.d i +на Ñĩ + ¿ +ig Decimal +Ġsh ore +Ġg ö +Ġrep airs +Ġhelp ers +Ġcenter ed +OL LOW +Ġmap StateToProps +Ġc ents +< A +Ġexpect ation +Oct ober +Ġbg color +ca les +.C ON +ĠV el +Ġcry ing +-se ason +Ġfunction ing +_LOC ATION +ü ss +ber y +Par a +omin ator +- le +Ġeth ical +has htags +emp lo +Ġn úmero +( activity +.St op +.str ftime +IL D +Ġto e +ĉ Node +") čĊčĊ +ĠPu erto +Ġexec uting +ĠG UID +Ġoppos ing +al ph +Ġexhib it +_fl ash +Ġme ille +Ġjson Object +H ero +aint ed +_D OM +Ġw il +Ġslo pe +Ġm Ã¥ +ĠIraq i +Ġorgan ize +ĉj Query +H UD +sh ine +. we +ĠSk ills +pons or +Ġcon clusions +Ġre forms +Ġrel uct +n amed +ĠOl iver +Ġ// }Ċ +- looking +Ġf og +ĠH O +ĠF ried +Ġinev itable +ĠData GridView +H our +il les +log ical +Ġconnect ivity +.tw ig +ĠK yle +(d st +- Sh +ĠStud ios +( Level +.j et +_PRO TO +-de coration +OT HER +Ġread ily +.Param eter +Ġmultip ly +ĠL IB +ar med +Ġsoon er +æ Ħ +_ ES +Ġfoss il +ĠA nc +âĢľ This +l odash +Py thon +Ġhist ogram +west ern +Ġinf ant +Ġco ordinator +Ġn ib +: m +Ġres pected +Ġdef init +& T +_p ad +ĠTr igger +th al +Ġimage Named +Ġbeat en +ĉ rc +ĠPal ace +Ġhaz ard +Ġisol ation +_ rc +cont re +OUT PUT +Ġre ign +ĠPl ate +AT ES +Ġfl ux +Ġpack s +.get Selected +Ġparticip ated +Ġneed le +-de pth +:::: :: +-l aw +ins pace +on itor += no +ĠAt omic +ĠBr ain +Edit able +-s c +red ential +ĠP erry +k ie +Ġ ----------Ċ +.st roke +( Intent +Ġun ity +um lah +F urther +Ġpr ze +Ġs ø +ãĤ Ĭ +ĠPROC UREMENT +ĠH ousing +Ġatt orneys +Ġcomp ose +atter ing +" What +dra ul +Ġstraight forward +In stant +.J TextField +Ġtr ades +л а +Ġ{ ! +Ġl ately +IM G +ĠA ld +ĠIN NER +Ġcart oon +.S ource +F ALSE +Ġd ough +f en +( rect +Data Table +N ick +ĠBut ter +read s +_com ments +EN V +ĠConnect icut +-F IRST +ĉĉĉ ĠĠĠĠĠ +ach i +.M sg +re ction +Ġrelax ed +Ġsha ft +Ġe f +ĠAdd ing +Ġbre ach +Ġ ï¼ļ +ram a +Ġconduct ing +Ġ( ; +(g l +ĠCA USED +ash i +ĠF LAG +ĠCom merce +ĠIN TEGER +h ours +ĠSchool s +Ġn ucle +Ag ain +pro j +Ġsevent h +EMPL ARY +(m ock +'] ,čĊ +_S PEED +> false +Ġsp a +ĠN ear +ì ķ +Ġintr ig +_m embers +w ave +Ġanalyst s +_O S +ed in +ĠF ri +Ġretrie ved +Reg ular +_ obs +EX PORT +')}} " +" class +__ (( +b ucket +Ġst ro +ĠP atch +yst ick +ful ness +ap os +D a +ĉĉĉĉĉ ĠĠĠ +Ġen rich +un ordered +h ole +C ong +< Product +ĠC urt +( the +_l ower +Ġavoid ing +Ġbu zz +Ġv iable +ub a +- is +are l +Ġact ed +-d etails +ภĩ +ĠThe ory +ĠP un +ĠAn onymous +... "Ċ +è res +åı ¯ +ĠV ision +_se m +ash a +Ġcelebr ity +Ġend Date +Ġpop ulate +Ġcu is +qu ant +f loor +Ġglob ally +Ġcru ise +ĠStan ley +Ġb ikes +.get Connection +Ġpoor ly +_ other +amp ing +." );ĊĊ +od i +_A DMIN +.color s +ĠG aming +> ';ĊĊ +STR UCT +Q R +ID s +(arg uments +_a ux +( Event +_PR IVATE +ĠTre k +Ġdownload s +m utable +_STR UCT +(w x +Ġdom ains +js px +ĠVi agra +Command s +J s +.c fg +Content Pane +ĠEdit Text +à¥į ठ+Att ach +ĠAR M +posit ive +ĠGener ated +Ġse ized += : +Ġelectron ics +ĠApp Component +/ ',Ċ +.equals IgnoreCase +Do ctrine +d isk +ĠPolit ical +CH O +< F +ĉ height +ĠB ug +. le +ik h +Ġmill iseconds +Ġconstit u +m ag +.n l +-r ange +ang gal +', [ +ropol itan +Ġà ľ +ĠU C +.d esc +-L AST +f stream +ib il +Ġf ier +VER Y +Ġë ³ +IR T +_ UI +( abs +Ġkne es +Ġro okie +ĠV ac +are na +comm end +- \ +ĠSUB STITUTE +So ft +Ġpart ir +we alth +è¦ ģ +(d ataset +ĠCl imate +- show +Ġreli ability +_ch unk +ä» £ +_st ock +ĠEX EMPLARY +ï¸ ı +Ġv ÃŃ +Ġsm iled +Ġdr ill +.F unction +ĠS I +Ġreg ression +- X +ĠJ ar +p ref +ĉs uccess +ĠHit ler +Ġinst inct +Ġfem mes +Ġlo ver +< Ċ +Ġmulti plier +r il +Res ize +ĠAuthor ization +ĠK an +Dispatch ToProps +Ġc rops +t okens +ec n +ential ly +ĠINTERRU PTION +f ake +Und efined +ĠA K +ĠTest Case +Ġr ab +Ġtor rent +ĠO t +B ars +Ġlect ure +Ġen jo +Ġrespond s +Ġindex ed +Of Work +_ch ain +)) -> +ĠBeaut y +Ġ` < +Ġtouch ing +Ġ| -- +ĉf lag +normal ize +Ġtr apped +Ġestablish ing +/b uild +A J +f y +- react +av n +RI PTION +Ġk ut +ĠF ashion +ĠIn form +cur ities +< byte +ĠUkr ain +Ġs ug +Ġconsist ing +ood le +. ctx +.To List +Ġcomment ary +Ġtransf ers +Ġn ost +ih ad +ĠU pper +Ġconf using +miss ing +- cl +Ġbound ing +Ġcongress ional +Ġreve aling +d h +r up +Ġt res +re peat +, ĊĊĊĊ +_t ac +Ġexp ed +G irl +h orizontal +Ġ"../../ ../ +( option +Ġwe iter +ĉs ql +Ġ=> {Ċ +Ġgar lic +Ġre pr +Ġrepl ies +( prop +Ġspir its +Ġins pire +Ġbas ement +.re ject +Ġhint s +Ġpoll ing +ĉ ĠĊ +_r ating +Ġc ath +av ier +Ġcomp ressed +ĠV S +] ' +Ġjud icial +ĠT rend +tr aining +EST AMP +ogn ition +Ä ģ +SE NT +vent ions +Ġconsult ant +um ph +Ġuser Service +, NULL +k h +D ear +_B AD +it ations +Ġmet aph +' é +and ise +-f ont +.ch art +Ġs g +_ Controller +.j peg +ĠUL ONG +ĉg ame +( ss +ĠM aj +ĉg o +ĠS ad +ĠB erg +ĠM ine +P ack +Ġres istant +ĠR OM +Ġp eg +ĠStan ford +ĠY ahoo +Ġsca led +Ġl an += [] +"/ > ččĊ +Ġs ud +ĉ background +Ġsch olars +-m uted +ar á +Ġ= ==== +Ġ__ __ +C reat +ene ver +/w p +ĠV PN +Error Code +) ],Ċ +(b uilder +ĠEn emy +S ensor +us a +Ġtr iggers +Ġplayoff s +_RE Q +Ġ( ~ +ĠBar ry +Ġperman ently +ĠR UN +Ġb ure +.Fat alf +Ġch ick +ĉ panic +ps i +ok a +éĢ ī +> [ +Ġunderstand s +ĠJun ior +ĠIN FO += mysqli +ust ain +-s ource +s erv +ĠC REATE +. au +Ġsell s +ĠĠĊ ĠĠĊ +E urope +z w +pre h +ĠNS A +Ġx y +ภ´ +ĠB eyond +Inst ead +Non Query +Ġar ise +Ġavoid ed +.em place +_model s +} ),Ċ +Ġh id +Ġ& _ +.p oints +.get Width +.Ex ec +Ġ// // +ĠS essions +... \ +ĠCol omb +Ġacceler ation +rest ore +Ġ ile +ob ic +< Node +ĠD X +ĠBes ides +. age +ĠCont ains +N ational +ĠIm plementation +Ġeff ic +ĠR M +H y +ĠWed ding +ok ies +Ġrec ursive +Ġprosec utors +.Se lection +ĠForm ula +Been Called +[i i +ĠFr an +Ġtraged y +_F EATURE +Ļ ¨ +comp ass +ĠB h +? ĊĊĊ +.w riter +ĠH our +Db Context +io v +am on +re pr +é ĥ +ĉf i +'] ] +ĠD ry +. ro +ĠO bserv +æł ĩ +Form er +ĠB alance +ĉ json +Ġpr zy +I SS +( sock +ĠL INE +Ġde ce +Ġal ly +Ġtend ency +F un +Ġschem es +Ġinter ven +æĺ İ +Ġad verse +quote lev +Ġsacr ific +_s ide +Ġmut ex +AG IC +Ġocc urring +ĠCommunic ation +um ar +ç¼ ĸ +ĠTreat ment +.p erson +ĠL C +Ġe ch +( (" +ĠDise ase +ä d +ĠA Z +.A ccount +Ġcontinu ously +END ING +ĠRET URN +- string +.f ilename +syn thesize +Res ponder +( opts +reg s +Ġn uest +Pe er +// ------------------------------------------------ +Ġg auge +ĠK in +.s chema +Ġarr ange +ĠBl ake +_Type Info +C over +ĠHamp shire +P aper +-in ner +util ity +Ġcross origin +F OR +Ġign oring +ĠD D +av an +Ġtrad itions +Ġget String +Ġeth ics +ĠMaterial s +DE SC +Ġen zym +io let +ĠCh ip +ĠMc Donald +Ġn erve +ç Ħ +") ] +æ± Ĥ +ĠS ugar +_S IM +j peg +Ġdiscret ion +ĠT N +bo ve +ĠMin imum +ĠForm Group +Ġwork force +ĠExec ution +err er +ĉ ĠĠĠĠĉ +Ġpres cribed +.Text Align +OP EN +ĠP B +im ity +ĠEx ternal +° C +ĠApplication Controller +Ġb arr +imp licit +_d ot +ĠCol on +C OLOR +.Pro ject +* }Ċ +pl aint +get Text +Ġindivid ually +Ġcheck box +U Y +ĠL amb +Ġdys function +ĠL ar +à ° +ĠCre ating +');ĊĊ Ċ +" They +loc ations +_C ORE +Inter action +umbn ails +ĠPart ner +b rit +Ġless er +ĠSl ot +set Attribute +ĠW ave +.p o +/ store +Ġbrows ing +_p d +sum e +s ed +Cur ve +Ġpl asma +Ġsusp icious +ìĿ ¸ +ĠB ah +ĠExp licit +_C C +.Client Size +\ View +Ġsub stit +lo on +ĠG AME +ĠB rid +Ľ 建 +_ User +Ġsqu ares +f one +Ġsac red +ug hs +] interface +ĠTh row +ĠK irk +Ġemp ire +Ġassess ed +T ax +ĠHe aven +-b uffer +_STAT IC +én é +-b ordered +Ġpun ct +(m ode +Ġke ine +S ent +ĠCal cul +ĠE ve +Ġsty lish +Ġoil s +.Test Case +Ġtrad emark +Ġliter ary +Ġconcentr ations +ĠRel ations +( Class +Ġstd in +Ġv æ +back up +. VERSION +.AutoScale Dimensions +st arter +Transaction al +- panel +St udio +k c +ĠCh amber +ĠSpi el +Ġr ho +ا ÙĦ +! ' +.At tributes +Ġmurder ed +apeut ic +Ġint imate +Ġtext Field +ĠBuff alo +d ummy +" % +ĠLib erty +ob ar +ĠT ank +ĠPop ular +erv isor +ĠIn iti +ĠM all +ĠP rior +C AP +ĠCl ay +ĠCert ificate +.L ock +-st rip +-dr iven +/ all +ĠMessageBox Buttons +_SE CRET +_p b +Ġr ats +ा ठ+Ġn t +.R outer +_top ic +Ġt ennis +ĠP UBLIC +ĠActiv atedRoute +Ġ' ,Ċ +Ġcost ume +Ġj okes +. Handle +ĉ byte +Ġflav ors +( cc +Ġperson as +ĉ image +ĠN azi +Ġgram mar +Ġú lt +Ġval ve +Ġv ic +ĠR achel +_in valid +P refs +std int +(r oute +Ġhtml specialchars +Ġpe oples +pl ine +Ġn v +ĠQu ant +opp ers +Ġcurrent User +ĠC atal +Ġrecon c +Ġconj unction +l x +amb urg +Ġinflu ential +d anger +ind ers +Ġ% @", +.config uration +os ome +. identity +Ġpick er +n ost +ĠDI Y +Aug ust +ab lo +Le af +ĠRec o +ck o +DO C +ĠH erm +: any +ĠInt erview +ĠT ex +x fe +( work +Ġle ap +He ading +Ġqu arters +\ Bundle +re b +Per haps +ĠG mbH +B irth +ĉ sum +ĠWat son +.n il +ç ¡ +{ }ĊĊ +ica id +Get ter +" name +Ġ" čĊ +_n one +z m +ac ute +uest o +Ġs ous +Ġre build +Ġnewsp apers +ĠH az +Ġk its +if o +Bl ur +Ġsu ited +- In +à ¯ +ĠKe ith +ĠNor way +IN IT +ire ccion +iet ies +_us age +ĠDou g +r ise +Ġtr illion +im ited +ĠR EL +al ic +Ġcritic ized +the orem +Ġce ase +Ġsid ew +ĠT erry +Ġsubs idi +Ġfirm ly +Ġaw s +Ġh ott +Ġdress ing +bad ge +ĠApp lications +è¿ ĶåĽŀ +Ġlaugh ed +Ġh obby +Ġmus icians +Ġ* . +. placeholder +Ġcount ers +ĠCap itol +SD K +Ġhel met +and box +qu it +Ġcriminal s +Ġteen ager +( update +G l +.se lection +Ġdis charge +Ġpresent ing +ufact urer +_UN KNOWN +Ġstress ed +å Ļ¨ +Pro to +_cor rect +ha us +Ġren ov +Ġfire arms +Ġtechn ically +-b rowser +Ġc andy +St roke +Ġexec utor +Ġocc urrence +ĠIP v +_INTER FACE +ĠRetrie ve +.b ad +Ex change +Nav bar +ĠK id +(get ApplicationContext +_ST OP +ĠB oss +List eners +Ġshoot er +ĠAl b +ä ch +Ġp ix +.key Code +al one +Ġabs urd +ĠC um +ĠNewton soft +ik t +Ġlaugh ing +Ġcapital ism +ree Node +T x +_QU ERY +.S leep +( login +Web Element +Ġcelebr ating +Ġde precated +Ġma ar +Ġart istic +_ASS OC +ĠBorder Radius +ĉw p +Ġsurviv ors +In ner +- red +Ġprosec ution +_ pp +(" $ +Ġcomm a +un checked +graph ics +r ors +G ROUND +( public +Ġcustom ized +ĠArk ansas +ĠR ew +Ġexp iration +× ķ +ĠC ul +Ġn ons +.F ilter +Ġsen ator +_def inition +ash ington +ym ph +/ J +Ġf use +ram id +ĠSup plier +Ġaut ocomplete +Ġ} ), +." ĊĊĊ +_function s +ĉ to +.e val +ĠT Object +Re ferences +Ġhe ated +H AL +Ġ)) }Ċ +} $ +ĠB arr +_UN IT ++ $ +Ġget Value +ip ed +ch ied +(v m +c ue +_int eger +_c ourse +th ird +Ġrevis ed +** /Ċ +_D IRECT +Out Of +(" ( +ĠFe el +Ġre ass +Ġsub title +per i +n f +Ġenjo ys +Ġtreat s +) this +-t abs +anc ers +Ġcontin ent +Ġcard io +S er +. question +Ġph rases +Valid ators +Ġpop ul +Ġl ÃŃ +s ong +_IN TERNAL +Ġadvis er +Ġp uzz +Ġambit ious +ĠT ob +ĠD P +Ġpres idency +Ġsurre nder +Ġwatch es +_b inary +ĠSo on +Ġcan ada +(" ")Ċ +] =' +ĠBr andon +eps ilon +r w +.add Child +.C opy +Pr incipal +Ph otos +Ġmarg inal +Ġbas ics +e ing +M ust +_ String +Ġo le +M agento +.c ustomer +(p rev +ภ¥ +Ġlo yalty +C og +Ġprot ocols +ĠCom panies +Ġtheoret ical +Ġaccess ing +ĠZ en +. ones +att ice +_w orld +z es +Ġtatto o +Ġmen os +Ġinter sect +"] ;ĊĊ +bel ie +Ġin active +.read line +-label led +.d one +lick r +ĠW ORK +Ġderiv ative +Ġd atabases +âĤ Ĥ +Ġs x +.is Array +Ġy s +Ġp ada +ĠBul let +(` / +is Active +ĠCG Size +(equal To +ĠColum bus +Ġmar ry +DE V +_l imits +ron es +I AS +Ġt au +min o +_W rite +ĠW ine +Ġ[ [' +ĠP ull +rit ers +ri ents +Ġsh ifting +up p +_TIM ER +ĠCondition s +Ạ¥ +ĠOr ders +ĠSt rength +æī Ģ +Ġvalid ity +Ġf ot +et ur +Ġb olt +åĨ ħ +ĠAl ong +os hi +Ġassum ptions +Ġmag azines +_S PI +Ġp unt +_PRO DUCT +Ġrel ay +ĠJ avascript +. te +- es +Ġwidget s +(f s +< Item +_ex tra +Ġrecru iting +E t +Ġnecess ity +p w +Ġnov els +uss els +Cre ator +ĠM VP +ĠO C +th ood +cl ients +)) * +Ġcharacter ized +_SE ND +ut i +T y +.from Json +@ Service +ãĤ Ĥ +Ch ris +_ Is +ĠJohn ny +Ġclean er +ĠInitial izes +UN K +( axis +еР· +ie val +ĠWar riors +} )( +DM I +âĻ Ģ +ĠTre asury +Ġfe as +Ġsl a +_EN UM +l hs +ĠIn stit +ipp ers +Line ar +Re ading +quir ies +-c ell +ch rome +.S earch +IN A +ç±» åŀĭ +ĠĊ ĠĊ +ĠSam uel +Ġmill s +Ġdon ate +ĠGe o +( rows +Ġshe ep +Ġé l +ä½ ĵ +Ġb em +_UN USED +ĠR CC +Ġintrodu cing +att a +ĠP riority +ĠF B +ĠSer ge +> "; +atch ing +ĠKnow ledge +ĉ The +; margin +less ness +op ard +um atic +() ));čĊ +Ġf als +(c ache +Type Id +éĢ ļ +_ choice +ĠGo th +ĠS ites +M G +_b order +Ind ices +Compar er +ĠRed istribution +Ġclo set +Ġvers atile +Input s +**************** **** +Ġob esity +qu iz +gr a +(g lobal +åĬ ¡ +Ġcollect or +Ġk or +ov able +AD C +ĠEvent Handler +. nc +Ġplay back +ient os +_p erm +_W ARNING +ĠOlymp ics +.n orm +ĠBroad cast +_sm all +dr ive +. iloc +Ġtyp ed +M EM +_con s +DM ETHOD +Ġl un +.d istance +(p ar +po on +Ġb ast +activ ities +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +: čĊčĊ +S ER +) && +_l st +ĠPol ish +Ġknock ed +Ġfrustr ation +au kee +Ġph osph +iqu id +_c oeff +æŃ ¤ +L atest +ĠD ust +T ipo +Ġmaint ains +Ġmar sh +inc inn +l bl +C are +Ġneighborhood s +_g pio +ĠAr senal +D em +ĠW he +_h ook +Ġl dc +ĠHar per +ĠBer keley +Ġgrad uated +Per cent +Ġarr iving +ĠAdvent ure +(s cope +(' * +qu arter +ĠMar ie +Spe aking +_code gen +Ġimm un +c aster +ãĤ Į +åķ Ĩ +ĠDim ensions +.rec ord +Ġtext o +ĠMich elle +P ending +( by +_P AR +uch t +be e +.Th read +amp ire +k now +ĠClin ical +Ġmargin Bottom +Ġdistingu ish +.F ull +. undefined +ĠSequ elize +################################################################ ############ +Ġeduc ated +_O VER +åº ı +ĠÂł ĠÂł +_e ach +Ġur ge +de part +Ġdon ors +ĠA u +Ġbill ions +Ġbelong ing +_ age +_ Int +Ġsub stances +m achine +!! !ĊĊ +Ġjson ify +ib bean +ĠC ad +Ġend Time +Ġc ycling +ĠUIT extField +Ġle verage +Ġvan illa +e at +La unch +( pt +st ates +ĠControl s +ĠRes pons +ĠJ ake +Ġas leep +fort unate +.next Line +Size Mode +ìĿ ¼ +Testing Module +G erman +ĠInvest ig +.re verse +ĠB ACK +( DateTime +Ġnon profit +ĠEx pect +Ġt anto +'] ), +ĉ the +M ultiple +(get Activity +_W AIT +Ġj á +de cor +lev ance +ĠGit Hub +min ation +_qu antity +.Sc anner +ĠL ion +éĶĻ 误 +Ġd re +Ġtan tra +Ġcontent Type +Ġf id +_ alt +NS IndexPath +- pl +åĮ ĸ +Ġantib iot +table s +ac ial +ĠReg istry +Ġol ive +ig ers +Ġsubscri ber +_p res +ĠSy ntax +Ġlo vers +. Byte +old ers +_for ward +al ways +C aption +Pr iv +ĠT ampa +is ateur +-labelled by +ĠTo String +Ġì Ĥ¬ +Ġinit iated +W F +Ġinstitution al +in ject +ĠSc r +Ġdo ctrine +Ġsp acious +is ure +ĠAn a +" time +ess aging +Ġc id +ĠN an +Ġin complete +T AG +-b uild +Dec ember +Ġres idual +(P DO +ĠList en +Ġg lyph +Ġg aps +ne a +.R ect +Ġsa u +ĠPhot ograph +Ġexec utable +ĠExp ert +Cor outine +_s izes +ĠN L +.is Valid +); }Ċ +- reg +Ġc iting +c wd +ĠOtt awa +ĠB att +Ġrenew able +Ġprelim inary +Ġas ylum +Ġw rist +Ġutil iz +Ġdet ention +F ast +Ġan ge +incinn ati +Ġste ering +ĠNa N +ios ity +/ page +Ġè ¿ +ster ol +Ġdis g +( DB +ĠDESC RIPTION +Ġ_ $ +Ġobst acle +Ġb izarre +Ġextr action +_ex pected +Ġlos es +ĠCele br +Ġhtml For +Ġexplo it +олÑĮз ов +XY Z +Ġmagn et +amp ed +Ġat oms +S ources +pect ives +Ñģ ли +Ġ= čĊ +Ġd are +ĠWal ter +Ġbright ness +Ġan notations +ë ı +is ke +S chedule +. images +ros so +Ġ" .. +g amma +Ġin structor +Ġover write +- am +Ġdevast ating +ĠSaint s +Ġh s +Ġbon uses +$ output +ij d +(Action Event +mon itor +Ġmatt ress +Jan uary +.j p +Ġcar acter +Ġim pose +_re st +ĠSign ature +Ġcoron avirus +ãģ Ĭ +_com pare +Me asure +it ated +el ijk +ig os +es ar +Ġrush ed +met ry +_SE PARATOR +_W E +_ATTR IBUTE +Ġy aml +Ġspec s +ĠR ah +ph eric +ĠInvest ment +ä ll +Ġappe aling +Ġview port +ç © +Ġmargin Left +Ġsub tract +ĠED IT +ĉ ArrayList +gr ading +ĠF ailure +as per +EE K +(n ow +< object +ĠAl ignment +ple ado +q tt +( ERROR +ĠIN VALID +Ġuser id +ra ises +ID I +Ġvari ance +ĠN il +/ delete +_M AIN +.T oken +.C ategory +> )Ċ +Coll ision +ĠGre ater +ĠR acing +al an +Ġmon etary +, new +ĠS orry +. Enable +ĠInstant iate +oll en +ë© ´ +ĠCall ing +_h our +AD A +Ġsh y +) ** +Ġ== > +Ġes pecial +Ġinterpre ted +! =" +Ġpharm acy +.s ingle +ĠC ialis +Ġpar as +.to UpperCase +ĠDem on +Pr ime +Ġrank ings +Add ing +_H ASH +ĠEx am +Ú © +ĠVict or +Ok ay +"] ;čĊ +Ġfort une +ĠF ETCH +exp and +.Inter op +Ġb arn +æ ¶Ī +ue vo +Ġspec ulation +âĶĢâĶĢ âĶĢâĶĢ +ĠN u +ĠBl ues +(f name +Ġinhab it +Ġ\" % +C ES +ular io +_c r +Ġvalid ated +Ġmid night +ank ing +Ġincorpor ate +Ġpurs uit +EX P +pr ime +P id +- US +ĠN urs +ĠW heel +é ĺ +Ġin p +Ġsupport ive +.m ember +ĠSh ot +.Check Box +Ġaff irm +T or +Full Year +Ġconsider ably +cred entials +_ opts +R oll +( round +Ġcom ent +_U ART +Ġext ending +R G +result ado +it u +.get Session +Ġattr action +& D +$ html +ĠJess ica +ĠAssoci ate +a ñ +_ ed +ĠL ag +Ġorig ins +()) -> +add EventListener +IAL OG +åIJ ¦ +.Com pare +Al bum +ĠK u +< Q +arg est +Ġpro long +Ġconfig urations +Ġaccident ally +_ph oto +Ġ'' ;čĊ +Ġver se +B ob +Ġfarm ing +del ivery +ĠM ack +Ġuse Selector +.bootstrap cdn +keep ing +en y +. upload +ĠM ETHOD +cre ator +< _ +ĠE aster +. -- +UI Button +ãĤ ī +om eters +Ġsh ine +Ġh ogy +\ s +Ġh arness +.C ell +Ġlif ting +Ġcomb ines +ĠOcc up +ex clude +pat ial +Ġres pir +_f it +Ġfif ty +ĠM ol +Ġtun ed +-d imensional +Ġq s +Ġto ps +> ";ĊĊ +quis ite +ch annels +/ res +ĠAn alytics +.app compat +/ to +Ġon Error +( attr +IR M +Ġrag az +- as +.Se cond +orient ed +Ġdon n +Ġlight ning +f id +ĠP le +ãģ¾ ãģĻ +t ro +.Tr ue +O bservable +× Ļ +umb ing +Ġpros pective +-f ilter +Ġpurs uant +(p oints +.B ind +Ġp alm +clear fix +ö s +ĠG onz +Ġwe aken +Dr ive +en ido +l ld +ob ox +ane an +G ot +ä¿ Ŀ +Reg ex +æ ĥ +Ġsal ad +ass is +" net +inherit Doc +ĠR V +qu ier +Ġcl azz +ı ÅŁ +oster one +Ġair line +.list dir +Ġdownload ing +ĠP alm +w aukee +& lt +.B L +_IN LINE +off s +<< ( +_new s +Ġch ase +/ >< +Ġeuro s +ĠEgypt ian +ĠSt ainless +_BO OL +ĠG uild +ĠD ynam +[index Path +Ġ ï +Ġmemor able +ĠCh ampion +Resource Manager +.Log in +ĠForm er +yp ed +Ġl leg +; ", +D WORD +Ġtax i +Ġbom bs +ra h +.t ags +_test s +st ones +âĢĿ ) +[ g +r type +Ġv u +Ġhost ile +Ch ars +ĠPatri ots +/ status +< B +ĠIn come +ĠD ad +Ġpat rol +_CH ANGE +Ġup graded +Ġch ina +set q +Start ed +.U ndef +Ġcheck sum +Ġfrustr ated +{ o +Ġen f +Ġwood s +ĠAny one +Enc ode +ĠQt Widgets +are as +Ġshe er +sk i +end point +_T est +S oup +~~~~~~~~ ~~~~~~~~ +(f iles +ĉĉĉĉĉ čĊ +.sp ark +Ġval ued +Ġ% Ċ +.control s +ĠXCTAssert Equal +Ġf ame +ĠR ic +D OT +ĠAlbert a +ä½ ¿ +os al +.Web Controls +Ġ ------------ +ĠM is +ĠS YS +Non null += item +Ġexp ire +Dec ode +_ operation +ĠValid ator +.C ENTER +uff s +* m +Ġav ant +æ¬ ¡ +âĢľ You +.per mission +... ) +ĠL ic +_co ords +.n ombre +c lo +.Int ernal +ĠCh o +_s w +ĉ Il +cl k +Ġcast le +(l ayer +p it +Ġgu ided +Ġâĸ Ī +Ġsuper b +Ġsup plements +_c ent +Ġpe ek +IN ARY +.Content Alignment +f alls +")) ; +W all +). čĊ +ĠD anny +irm ingham +IAL IZ +( create +" In +Service Provider +Ġpr iced +mac ro +am ac +. box +---- Ċ +ãĥ « +ĠS uit +ur st +br u +ourn als +num ero +__ ()Ċ +D as +ĠM itt +ud er +? \ +f u +[ B +Ġ: )ĊĊ +(int er +br ains +Ġatt itudes +Ver ify +Ġsign atures +ack Bar +Ġg d +J ack +.c at +Ġz z +war f +FT ER +");ĊĊ Ċ +Al ive +IC LE +ĠWh atever +Ġout lined +s prite +еР² +_A B +_DE PTH +Ġcrush ed +aa a +(e v +æľ º +Ant i +IC O +is EqualTo +.s un +ic ulo +s ale +_h ex +ĠV k +apt or +Un ion +ĠDis count +list a +.Undef Or +Ġautom ation +N or +å¯ ¹ +åı Ĥæķ° +Ġref lex +ĠLa ure +.showMessage Dialog +.t emp +Ġa kan +Ġ__ ____ +.Is True +ARE D +ag le +E nergy +Ġquant ities +âĢĻ é +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġcitizens hip +m outh +Ġin appropriate +ĠOut door +White Space +An onymous +load s +webElement Properties +T en +Ġacc idents +Ġadvertis ement +ĠY emen +(c all +Ġsl avery +Ñģ п +ĠL am +_BIT S +ome ga +ĠO le +Ġkid n +_A n +ĠR aid +Cre ation +s aved +Ġpro port +W ARNING +\ P +Ġp wd +Data Reader +is cher +ade on +ĠP redict +Ġreason ing +Ġdestroy ing +H el +* d +ĠLeg isl +_P r +ĉĉĉ ĠĠĠĠĠĠĠ +Ġsymp ath +Ġch ess +Ġm am +: hover +Ġconvert s +Ġp ela +Ġprogress ion +Ġ"_ " +ĠG ill +ĉ show +Ġsupposed ly +ac curacy +el in +Ġunf olding +ĠHy per +Ġw anna +Ġup s +( # +ĠCr iminal +( Point +at Lng +act ly +Ġcontract ors +'] } +draul ic +ód igo +ĠT T +ĠW ide +ĠAR G +_ ic +FLAG S +S chool +Ġclear ing +-be ing +={ [ +, const +man ent +Over lay +(' " +éĩ ı +ĠT imestamp +Ġmail ing +ĠC ake +.Th at +Ġmed itation +q p +Ġemp resa +ĠL ions +Ġw eld +ĠLinked In +Ġc ush +Ġgen ome +.Index Of +ag ain +Ġf allback +Ġcamp ing +re dd +-strip ed +Ġd v +Fe bruary +ĠPro xy +us k +Ġdies el +W RITE +RE AK +L orem +.In voke +- div +Inter ceptor +ĠD H +ia les +Ġvill ages +Ø ´ +ĠEN V +S ys +.X R +Ġpo em +à Ĥ +c ade +pl ots +Ġ{ ( +.g it +/s vg +nc mp +ĠÄ į +ain es +åĩ ½æķ° +Ġ( )ĊĊ +ops is +ĠRel ationship +_ aut +ĠB omb +ĉ com +* sizeof +off icial +_p ayload +ĉĉĉĉĉ ĠĠ +.m anager +ĠA round +ĉs end +ĠEx ercise +ĠB illy +iv i +Ġneed ing +_url s +_t asks +ĠH em +Ġtear Down +enc rypt +.t ie +Ġas m +IC H +ĠCGRect Make +ìĦ ± +ul ong +Ġit r +ĠG ST +Ġoffer ings +ro be +EE E +oper ators +_PRO P +ind ent +A DE +or f +ë IJ +Ġbless ed +vas cular +Ġcon oc +H appy +B ridge +ilit ation +j oint +ĠAdmin istr +- transform +Ġmeant ime +/ K +ĠBed room +Ġrig id +Ġbrows ers +EM PTY +.S erialize +_ ED +Ġst itch +Ġj an +ell t +Ġbr ace +Ġtr ails +p ublished +å¯Ĩ çłģ +} ')Ċ +Ġac ids +Ġ! !! +_d irect +> ());Ċ +aj Äħ +_O CC +Ġplan ets +æ Ł¥ +ĠDub lin +Ġser ie +.print f +de ep +` ) +Ġ\ $ +ĠÎ ¼ +_V IDEO +end ors +ĠC rypto +F ar +.Trans parent +.T R +ias m +_tr aining +Ġteach es +ĠB elt +Ġlimit ing +ĠK ath +ĠIndex Path +Ġachie vements +Ġser á +interop Require +Ġdis se +.I f +arm ing +uls ion +P o +_DE TAIL +Prot otype +ĠC AL +Ġagre es +.v o +.Execute NonQuery +ĠTop ic +Ġ' {} +Ar m +Ġe cc +M ag +Ġserial ized +ĉ conn +c ached += tf +ĠByte Array +prot obuf +var char +ĉ ASSERT +Ġlist e +_tr igger +· ¸ +Fe el +T ahoma +ĠL ik +Ġstruct ured +erg us +.In itial +_ ge +cl js +.cont act +Ġand ere +$ stmt +_C URRENT +ĠDis cover +$ res +form atter +H a +vang st +Ġem erge +ãĢĤ âĢĿ +ĠCabin et +-s quare +éĥ ¨ +Ġr age +ĠA J +ĠV T +sh adow +ĠFa ith +en ames +pret ty +has il +part y +Ġvar char +Ġf otos +Ġal um +ĠBelg ium +.y label +Ġde j +_num bers +Ġh u +.set Adapter +ĠUs ually +(s ample +.Sh ared +Ġbook ed +Ġ>> = +Ġmin erals +"> +pro g +bo o +_m d +_p ack +(ex press +ut z +\ Auth +, id +ĠCh ile +act ice +Ġrecruit ment +Ġpos es +Ġvulner ability +inst anc +or um +d ess +Ġx l +%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%% +( fig +Ġdelet ing +.d el +) ')Ċ +ĠWeek ly +?? ? +(str cmp +sm ith +Ġpurs uing +- so +ĠApp s +/ 'Ċ +Ġdec is +FO RE +Every one +Ġl anes +V irtual +. attach +( Log +ĠMed icaid +( Path +ĠTurn er +/ application +Ġport rait +Ġopp ose +check out +Ġfinish es +_M E +Bar rier +S ong +V AR +Ear lier +rell a +Ġh ast +az ar +Ġpull s +ng x +Ġinspir ing +Ñĥ Ñİ +-d irection +Ġexplos ive +Ġcreated At +st o +Ġwhe at +ĠB uilt +' ai +Ġtrack ed +ham mad +RowAt IndexPath +_ heap +D ue +Ġconnect s +.p ublish +em u +Ġbul lets +B AR +ol ate +Ġintern ally +Ġcatch ing +-p assword +ou ched +æĢ § +e ous +Ġx range +Q uality +v v +Man age +( ($ +ac ements +ĠBro thers +ĠHE AD +ĠUn supported +s an +es i +** *Ċ +Ġadapt ation +ĠWork er +'] / +.save fig +( trans +Ø ¬ +ne e +Cor rect +... ")Ċ +Ġsubmit ting +-p ath +ĉ last +iss an +.x label +ĠS epar +/ no +_b est +ĠM ills +_s ock +(f lag +Ġdest inations +em ption +ĠF AIL +å ĴĮ +Ġr p +f act +ĉ len +D AY +Ġse iz +_d st +l ip +.Line ar +ĠB asket +$ t +$ i +- brand +ĠNe il +ĠE q +Ġth ou +og ene +Ġscholar ship +æĽ ´ +Ġs wo +ag inator +en i +( book +Ġbl ink +th us +Ġcancell ationToken +ĠPalestin ians +Ġprofit able +Ġback pack +ens on +< Long +Ġp ools +Ġst icks +Ġspokes woman +Be ing +ĠHer itage +ĠN ike +SH A +ĠNotImplemented Exception +$ core +ĠR ico +/ latest +ĠC zech +ner Radius +(l ines +Ġsem ester +Ġw ounds +Pro cedure +.m ail +() ):Ċ +Ġcor rid +ter ed +ĠN CAA +Ġgal axy +_k ind +il k +Ġtr as +_P OL +ĠH et +Ġrefuge e +Ġteen age +.b inding +post al +Ġiç in +ĠData Type +é ĸ +ycl erview +, value +_id entifier +< b +Ġout file +čĊ ĠĠĠĠčĊ +Ġcr é +Ġrespond ents +ĠBe ast +ce led +Ġinter f +-th eme +g if +ĠR angers +IT AL +Ġauthentic ate +Com pletion +urs ors +Ġcin ema +Ġdisc our +ĠJ aw +OCK ET +Ġpr ayers +ĠL uis +fr ag +=[ Ċ +Ġbr ave +_p ose +C ertificate +- fe +ifer ay +ĠFl ags +Container Gap +ĠC rit +Result Set +ĉc ur +Ġcorrespond s +St aff +.Http ServletRequest +Ġneur ons +ĠMain AxisAlignment +ed ar +Ġg ad +_p arts +ĠÎ ² +Ġf x +/ files +ĠB ros +hip s +Ġgluc ose +Ġfar ms +Ġment ally +rest aurant +Table Name +ĠMer cedes +. Visual +Ġan ch +inal g +_r untime +Ġpropri etary +Ġintent ions +iz i +S lice +; "> true +ĠNY C +Ġb ored +ĠD etect +Ġapp ar +Ġje ans +ĠT ak +I OD +ĠH orse +( FILE +( ? +ri que +optim izer +n at +lo ys +ĉ Token +oub ted +u ess +oco a +Data Member +_P OWER +class List +Push Button +ĠWi Fi +. Stream +.g uild +Ġn og +ĠPortug al +ĠUnt er +Pr imitive +b oss +ĠDe utsch +Ġerot ic +Ġstr conv +.Try Parse +Ġgr ams +.S uccess +_p k +ĠHar vey +-m inded +.c ountry +[] " +Ġang el +Ġbe ats +ĠV or +il io +.m aster +s omething +ĠP ACK +( if +Request Body +Ġant es +/w idget +Ġmod o +ĠA W +find er +Ġoptim ized +Ġmiss iles +N B +ĉint ernal +t ex +ĠS ri +Ġdam aging +ĠM ais +- Allow +ĠZ h +- alt +Ġ ));ĊĊ +è ī +Ġinflu ences +Ġc atal +_REG ISTER +ĠAPI s +-cent ury +Ġbi ology +ĠAct ual +Ġhe els +TR ACE +_D IG +D ataset +ĠM atter +Ġclass ifier +.w ikipedia +ĠRog ers +Ġdon ated +raw ler +en en +Ġcas inos +ort al +Ġpr ive +s pe +duc ers +. ep +Ġgr asp +ac ji +Ġd airy +Ġb uses +.com m +. ins +ĠI RS +ĠBe er +ad c +o ard +_M ET +Ġ' +' +r ans +Ġkind a +ĠâĶ Ĥ +ĠM aur +аР³ +Ġband width +ib us +ĠD ifferent +(m at +ĠRes ume +_UN S +est ablish +Ġfon ction +Sub scription +_com pany +Ġlight ly +.con firm +.y aml +ĠBo ost +Com merce +- template +_DEL AY +ĠH I +Ġn avig +(S ender +ĠH S +_ "+ +ĠRE QUEST +Ġw ifi +=" "Ċ +]) -> +Ġro pe +Ġviol ated +Ġgl ance +ĠK urd +Ġè ® +de ck +ĠIS BN +Ġin fect +ĠF oo +Ġget ter +Ġt ener +ap pe +.h h +_h ot +< AM +p oly +! ",Ċ +Ġconver ting +ĠW WE +RO S +(' { +Com mit +) L +ĠO re +Ġsp arse +Ġdis posal +Ġcan celed +åIJ İ +Ġa er +Ġvin yl +á» ĥ +rec ogn +ark ing +Ġtrick y +* s +Ġproceed s +Ġis o +Ġco conut +Ġcraft ed +IEL DS +Ġquest o +Ġcomm un +_CON NECT +Ġtraff icking +De ep +a ções +c odigo +ve au +Ġbet ray +int a +T ED +æ r +m art +_B US +/ sc +ial ly +Ġcigaret tes +è¯ ģ +(n n +Ġmodel ing +/ products +w arn +Ġmet ro +ĠI v +& ) +ĠC able +Î » +Compar ison +g ary +ĠB A +P ART +Ġp v +_up dated +C redit +orth y +observ able +Ġthe atre +B LE +; }ĊĊ +la unch +_str ings +ug o +ĠR PG +- auth +Ð ł +hol m +ĠP and +U id +Ġim ply +ìľ ¼ +'] =' +/ User +Ġstr cat +нÑĭ й +Data Adapter +Ġland sc +Ġdipl omatic +ï¼ ĵ +************************************************************************ **** +ĠCh icken +Ġbc rypt +.In f +[ col +ĠQu antity +- position +Ġdiet ary +Ġfil mm +Is rael +Pre v +ĠMill ion +Ġrem ed +Ġbill ing +Ġout doors +.t m +Ġn ad +F org +Z Z +Ġs sl +], ' +K T +f req += document +bl ur +¬ ¸ +ĠJeff erson +C s +(s ave +Ġstr ap +Ind ia +Ġide ology +BO SE +ĠF P +( ans +Ġfe ver +ĠY am +K ing +à ² +AT ING +bo hydr +roll back +Ġnew Node +ĠN VIDIA +Ġhon our +ĠCon firm +xb d +Ġsuccess or +/ u +l iv +ourn aments +Att achment +Ġgr up +Ġtri be +Ġca res +e ft +_s ame +' label +Ġ ãĢIJ +M otor +Ġin exp +Ġ" (" +_POS ITION +Ġval ley +ĠResult Set +Ġpres erved +Ġmut ations +Ġquestion ing +mun ition +parse Int +ĠS r +ĠMet adata +âĢĿ ï¼Į +timestamp s +Ġtrans itions +í Ļ +Ñ Ĭ +i om +.D o +Ġp ine +Ġf ung +Ġtrans mitted +ct ime +ĠF am +Re vision +B as +UP ER +D estination +toHave BeenCalled +Ġun fortunate +IN ES +_pro f +Am ong +ĠCy ber +ĠB attery +gen re +ĠView Model +- = +Ġutil ized +p aint +.Integer Field +ern ity +comp iler +âĢĭ ĊĊ +ĠM asters +.To Array +Ġstrt ol +ĠUkrain ian +} ));Ċ +Ġsh emale +" That +for all +/ download +Ġrhet oric +.l atitude +ĠWH EN +Ġshock ing +IF IC +.N ormal +_F OLDER +Ġdr ift +Ġmount ing +- book +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ĠWire less +> ".$ +Ġrel ies +( Console +Int ernational +-> {$ +M id +Ġdis sert +dd s +Ġdepos its +ĉd river +# ga +pr ising +print ln +Ġpres enter +Ġmin es +C SS +ĠD ual +(! ( +Ġk am +Ġis Loading +ĠProt ect +. upper +ar ium +]: ĊĊĊ +Y ii +-sh irt +ĠIM AGE +_color s +Ġur gent +.Cont ainer +! (Ċ +S aturday +Ġsoci eties +ĠTh an +ĠC od += @ +Ġattach ments +.m obile +Ġsp ite +Ġb ounce +raw l +instanc etype +ĠTr uck +Ġmanip ulation +( Config +-in st +Ġst or +it ution +Preferred Gap +Ġmain AxisAlignment +Ġlist ened +'' 'ĊĊ +ott age +- project +.AP PLICATION +ĉ root +Ġwh it +Ġb ilder +Ġk er +Ġappl iances +row ave +ìĿ Ģ +ematic s +ĠO rg +op ing +_SE ARCH +Ġch am +add ContainerGap +Ġ( ). +ĠAr row +Il legal +Current ly +Ġus a +Ġpassword s +Ġre nown +av ern +ĠEv il +Ġconc at +Ġdu o +Ġv ale +ĠBe an +Ġindic ators +cm ath +ĠP ump +Nov ember +ific ant +_DOM AIN +reg ar +ĠPort al +" $ +Ġformer ly +"] :Ċ +ĠVis ibility +.getElementsBy ClassName +_RE D +Ġch ampions +à ´ +Val or +_ es +* a +-re peat +B and +.st age +Ġbure auc +C nt +et en +- function +Ġm uito +P ID +_ editor +Ġcrash ed +de ad +k at +ag h +ĠEX T +ass er +-sm all +Ġreal iz +( Entity +ú s +ĠAct ually +ĠEl ite +Ġhel m +(non atomic +ash er +Comm unity +all eng +ir y +ĠG rowth +Ġs ue +Ġfrequ encies +_des criptor +.At tribute +Ġrecip ients +_N S +/ "+ +ib an +Ġath lete +ĠI gn +_D MA +(d s +ĠRequire ments +AD I +ere z +\ Admin +br aska +ĠR ust +Rel ation +C OD +ĠV ERSION +em ma +)) { +.D uration +ĠC amb +- logo +Ġread able +Ġcre ators +() ];Ċ +Up Down +-h alf +.get Month +(s f +P ic +Ġhun ger +.t x +Ġexceed ed +_se ed +( ^ +_s k +.per form +Ġ> :: +Ġm ongo += float +bind Param +Sm art +if a +Ġse curities +Ġpre jud +Ġ, " +Ġcor ps +Ġv ra +amac are +it err +(M edia +uch e +Ġc ob +Ġlib er +. geometry +Loc ator +Ġsl iding +Ġsurg ical +_C UR +Ġcon sect +[ * +ĠRes ort +St ub +_DO UBLE +ĠS oph +Ġelect oral +_dis able +ĠÑģ о +ĠLight ning +Ġment ions +oc y +Ġle aked +Ġrelax ing +Pres enter +v sp +Ġgu ilt +=- =- +.re ply +ĠMir ror +C amp +Ġ+#+ #+#+ +Ġ+#+#+#+ #+#+ +.A uthor +Ġdirect ive +-h ook +íĦ ° +}ĊĊ ĊĊĊ +@ pytest +_r and +m is +Ġcolor ful +u je +lass es +ĠClass es +.h ave +% ), +é¢ ĺ +Ġdistur bing +sub string +ĠK oh +In vest +p urchase +Ġrec ycling +ĠA RT +ier archy +Ġf ps +.check Box +íķ ´ +_m aterial +duc ation +Ġf w +ud it +Ġreview ing +ĠS id +S yntax +ĠW ritten +arg ar +UM E +/ q +Class ifier +Off icial +Ġj azz +Ġom ega +Ph ysics +Ġl ugar +_access or +.command s +Ab ility +ĠB atch +R AM +Ġencount ers +. Qu +BY TE +ĠD istribution +Ġus o +ĠReco very +appro ved +Ġden ial +/sh are +Linked List +)čĊčĊ čĊ +udd y +Ġf ines +Ġr y +Un icode +ĉ render +Ġprem ises +Ġp on +ali ases +/F oundation +c uda +ĠC ock +,: ) +(f older +Ġm éd +dr ag +Ġtal ents +ĠĠĠ ĊĊ +е ÑģÑĤв +m ob +.y ml +Ġa ster +Ġdis cre +go al +ĠGT X +ĠS UCCESS +ĠL ONG +(f ind +Ġsing ular +_s z +ĠEth ereum +.. Ċ +Ġir res +')) {Ċ +Ġmin isters +St eps +ivers al +ĠNever theless +- led +Ġ( %) +ç¡ ® +Ġtime zone +Ġstr anger +(re nder +Ġsh util +Ġm ph +Ġtri o +pp y +Ġpred omin +Ġend ors +ĠRuss ians +ĉ row +Ġw izard +.s erialize +Ġcompl ained +Ġs ido +Ġdelight ed +-m e +ĠR av +H uman +ad ays +rec v +Work ing +J ump +ĠÃ¥ r +ĠAut omatic +_B ase +æł ¼ +aur ants + ¯ +æ ¸ +(C Type +IF I +( amount +Ġbelie ving += mysql +Ġf ir +Ġrest oration +ere co +Ð ¢ +_ '+ +Ġe book +Ġde bris +(input s +AY OUT +Ġscre aming +av ia +land er +Ġdist ress +Ġas sembled +ĠA void +( thread +ĠR PC +_EX IT +( queue +и ÑģÑĤ +D ll +Ġsk ull +_p ub +che z +min ate +ens en +Ġins ane +b ounds +ĠR osen +Ġcondition ing +process ed +v ideos +f our +.Con v +| ;Ċ +Person al +cer pt +:UIControlState Normal +Ġdos es +ĠKar l +ĠFre qu +.B ASE +ĠV ote +Ġcon current +ĠMessageBox Icon +Ġà ĸ +ĠDub ai +ĠR etail +: number +ĠOb server +ĠBig Integer +_ origin +_W ORK +F rames +Ġnot ably +. âĢľ +Ġtrop ical +Ġn iche +am ina +.s ys +(t okens +mod ify +os it +st rom +ĠCom ics +O PTION +T icket +Ġfact ories +Ġdis put +_F ile +ĠFin n +ee e +ĠDisc ord +_m oney +.t pl +_s afe +L B +Ġgl ut +J K +.fl ow +- cont +g os +Ġhor izon +ĠR ush +:: * +P ipe +ull a +bor ough +he imer +(m ove +( Text +} );čĊčĊ +w elcome +ĠCom ponents +Ġgovern ance +c losed +ĉm argin +Ġla undry +ĠTerm inal +iz ards +. âĢĶ +.rem ote +.r adius +ĠQue bec +Ġd h +T ech +ĠM ist +s eller +_l iteral +Ġgen ius +Ġbr ains +g em +ĠMe asure +Ġcata st +r ance +.Text Field +Ġconsum ing +Ġ'\ '' +oubted ly +ĠC ertain +E v +ert i +be ing +Ex perience +Ġ// [ +ĠArab ic +ĠC rist +ĠAz ure +Ġhor a +l adesh +\ Blueprint +d ar +.re l +Ġsup rem +ĠRe agan +ĠAt tributes +-s idebar +Ġuse Styles +ĠA irlines +Ġh ills +/x html +v inc +_m ock +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +ĠP ill +.Layout Style +ĠCommand er +] < +sign ature +Ġ{ }čĊ +Ġhat red +Ġë ĭ +ole sterol +Ġ ******** +ancell or +c rop +T IM +ĉĉ ĊĊ +ys qli +uit ive +ĉun set +_s el +Ġmen us +t ick +Ġconstit ute +ĠElement s +ĠRed is +agg io +_f p +_de pend +em as +CA ST +or ange +j on +ĠEm ily +Ġpot atoes +Ġre ceptor +ĠElect ronic +ĠL ights +Ġcomb ining +ĠSome one +Ġ######## . +ĠT OD +/ show +X d +." ' +af x +Ġtr agic +St yled +ĠMar co +G allery +d ale +.âĢĿ ĊĊĊĊ +é rie +/s ervice +äº Ĩ +Ġamb ient +_SET TINGS +.Ad apter +l ene +Ġtrav els +Not ice +Ġcle ans +ĠF em +ch air +Ñĥ н +/ my +_b ad +ĠEcon omics +IS A +_C NT +(M enu +äº İ +ĠR idge +Ġlength y +D ot +Ġjump s +Ġhe y +$ pdf +Ġw orm +Ġs ut +Ġsh er +iam o +ĠCal c +trie ve +Ġc ops +ĠCh rom +Ġreg ulated +reat ment +ĠHigh er +ok s +Ġde ze +LOC ATION +ongs To +Ġfin ite +Ġvar ies +Ġposition ed +' il +éĩ ij +Ġh ike +(d one +play list +Ġad a +Ġcoast al +ĠN ancy +.DateTime Field +Cpp CodeGen +ĠSimilar ly +re ur +ĠCon tr +ĠH idden +ĠB eta +atch ed +_inst all +. Output +Look up +ĠRich mond +qu ared +Ġm anga +-control s +ĠBern ard +L arge +Ġslic es +Ġoff ence +ĠM ega +Ġest ar +Ġjoint s +Ġsum m +_pl atform +B uff +.add Subview +Ġret ained +Let ter +.d im +Ġess ere +ĠS caffold +EX PECT +ĉ RE +.long itude +ü nd +Ġstat ue +.add Widget +ĠCar ibbean +add PreferredGap +il de +UIL abel +ĠOp port +Ġimper ial +urs ion +Ġmand ate +Ġpromot ional +Ġv k +ia ÅĤ +Ġp yl +ĠCre ation +оз д +Ġsim pler +. what +ĠRec ent +St orm +. quantity +ĠL ov +" - +ubb les +_not ification +(w orld +ur ger +* (- +: "Ċ +h m +ans hip +ĠAl most +Ġmotor cycle +_f ee +Ġabsor b +ĠVin cent +Ġsound ed +ÃŃ st +Ġpharm aceutical +ht ag +ĠKind le +ital ize +ĠEm peror +oust ic +Ġspecial ists +åħ ¬ +Border Style +/ \ +RE LATED +(', ', +(ex pr +Ġh t +åį Ī +_C reate +Ġspecial ly +Ġ[] ;čĊ +Ġhe el +Ġse pt +_ arch +(in itial +% .ĊĊ +\", \" +Ġdiscuss es +Ġu pt +Ġ[ & +Ġman us +.h and +ĠM AIN +ĠDen mark +Ġ], čĊ +Ġcr yst +Ġn ack +Co ords +_in ner +Ġmid st +Ġaw ake +ĠÐ ŀ +-b reak +ÃŃ vel +_P ASS +ĠParam s +Ġdet r +Ġsp ider +ĠCon cept +Ġpre nd +CH ED +.Ex it +Ġpop ulated +Ġvirt ue +_SE SSION +Ġnou vel +o auth +Ġд аннÑĭ +r ink +.Header Text +atur ated +Ġer st +Ġå ħ +ॠĩ +_vis ible +ey er +Ġli able +Ġde be +Ġb w +{- # +_W IN +df s +H over +ĠP UT +- angle +Ġnob le +Ġtr aces +enc v +Ġuser Data +_in s +ĠS uz +Ġnews letters +ĠMod i +Ġentreprene urs +Ġtrib ute +Ġrum ors +Ġr r +ĠQu arter +ê³ ł +Ġfeed s +ó g +Ġen velope +Ġle ar +Ġk ø +develop er +Sim ilar +: ")Ċ +sub scription +Mod ifier +ital ic +Ġn asty +Ġtermin ation +Ġchar ming +Ġâ Ł +ton s +.tr ace +h ots +ĠU R +M ont +Ġjust ified +ĠG ang +ine a +Ġb og +( ap +_ $ +Ġcont amin +.D ot +ĉ Debug +( exports +Ġpa ired +ĠAss ignment +Ġautom obile +ĵ į +Ġph ases +v w +@ SuppressWarnings += \ +r ant +- ed +ĉ await +Ġcert ificates +'> " +Ġint act +CT RL +M ike +greg ation +AT TERN +Ġre public +_up per +ili ary +Ġcomput ation +h ire +ĠSh in +_ ANY +ĠManufact urer +ĠC arm +Ġbear ings +_c omb +c ad +ur istic +Ġwholes ale +Ġdon or +.inter faces +press o +ĠBr un +-c lose +pro ve +_S K +ĉf rame +et ros +ĠP ain +_EX P +ĠL T +_f s +.dat as +ĉ ss +vo ir +ĠA xis +M ajor +=" < +[ h +Ġprof ess +igr ate +(s core +Key word +" os +ĠĠĠĠ ĉĊ +an alysis +Ġre play +.p ass +\ d +t ls +Ġsan ct +.l ight +_m obile +ÑģÑĤ ÑĮ +ĉt otal +u ity +Ġpa used +N AS +Ġen core +lo e +Ġ-* -ĊĊ +.h igh +am pler +ĠSec ure +Ġfrag ments +_ vel +ill ary +ĠSte in +ĠD awn +Ġmax imize +ภ¢ +Ġ/ ^ +Ġcontin ually +Ġsh adows +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠI ActionResult +Ġinform ación +C HECK +.Selected Item +b undle +ol ley +< Int +AIN ER +ĠW ing +tit les +ount ain +C Y +ĠLoc ale +form er +< context +R adioButton +_s chedule +Ġfab ulous +Rob ert +_PRO FILE +Ġg ates +IM P +ĠPent agon +g old +b ach +employ ees +R otate +Ġch amp +Ġsel bst +Al tern +Ġconvert View +/ , +Ġ~ ( +St reet +_ place +Ġpersonal ized +P ublisher +ĠSO CK +_NAMES PACE +ĠStand ards +so ever +_C ENTER +Inter est +ô t +tem perature +View port +get Resource +Ġeat en +Ġsem pre +Ġab normal +Ġc ylinder +Ġtroub les +n od +Ñĭ в +g ames +_g l +Pl ane +g rey +_t bl +.Component Placement +ĠCh ase +Log ging +man y +ì Ĩ +Ġfl ame +="< +Ġtra jectory +_r ing +Ġhydro gen +tr on +Ġstat ute +Ġcondition al +Ġtr ay +-s chool +(w idget +$ config +Ġrequest ing +. uint +et on +brit ies +Of Type +AD MIN +p redict +Ġg egen +ĠH app +OC UMENT +ĠA part +Ġ---- - +ro e +u ide +just ify +ĠSqu ad +Ġprof es +.b ot +_c urrency +inn en +ĠM umbai +ĠNum bers +avana ugh +agn itude +âĢľ There += http +çī ĩ +Ġv b ++' {{ $ +Ġin ode +s il +Ġh ace +Ġsever ely +ĠOver view +Ġspr aw +Ġbeach es +: left +· » +($ { +ĠF IRST +ĠSp a +- ass +Ġb aise +ĠN ODE +ĠP izza +P et +(se q +\ ">Ċ +CppMethod Pointer +Ġv p +Ġi a +_se conds +em et +/b lob +_TH RESH +... čĊ +D est +ĠN H +.data Source +it és +ĠJ ak +s ell +Ġwork shops +< u +Ġr ivals +ĠEX ISTS +h om +-t oken +compat ible +.J Panel +Ġphys icians +art in +Ġdes irable +Ġdistinct ive +.D ep +g id +ili ate +, max +Ġprem iere +Ġq Debug +Ġadvoc acy +Ġwh isper +P t +Ġun changed +_q ty +请 æ±Ĥ +Se ason +avel ength +ĠP ul +Ġd ÃŃa +'] ]],Ċ +al is +(" & +bor o +Ġb m +ĠR adi +w rong +ĠGo ing +ime Type +ij i +- feedback +ĠN ames +ĠB apt +Ġprob able +ĠE ther +ĠPolit ics +_prot ocol +lin ing +S at +Ġcor rel +.Pr imary +(null able +RI ORITY +Ġcolor ing +Ġutil izing +d as +Ġexport ed +Ġcar riers +Con v +. editor +i ó +(h andles +Ġapprec iation +. import +ĠAust ria +ĠStr ip +il ight +Ġappropri ately +ĠP rest +ĠW ir +ĠUI Application +al chemy +ĠM ob +ĠD etermin +ergus on +register ed +_con vert +ĠVlad imir +.Show Dialog +ref lect +Ġsh ook +Ġass ure +ĠO ften +Ġcivil ization +Ġvocab ulary +fore ground +ĠS cope +Ġunw anted +act ing +Ġ( [] +Ġmark ing +. original +ĠMO VE +Ġsport ing +ception s +NS Number +S izes +Ġprovinc ial +_Tr ans +Ġproblem atic +d igit +ĠEm ma +lock s +ĠC rew +ib a +') : +ish a +Ġm amm +Ġocc ured +w cs +(r ule +Ġmerch andise +es pecially +ĠT win +Ġn aming +Ġs log +Ġimpro ves +Ġad her +: text +.h adoop +_HT TP +.to List +.dis abled +Ġl enses +.in i +ĠR are +ĠUb untu +Ġsc ram +ol ation +tit ulo +Every thing +Ġnod ded +icht ig +_const ant +z c +l ift +ĠNot ify +ond o +ĠIN F +(" + +ĠK az +Ġd read +.m apper +le ur +ĠCome y +ĠN B +ic ers +.P ush +ĠH ack +ĠBrazil ian +_pro d +Ġ// ĊĊ +Ġb icycle +Ġun available +Ġadoles cent +bl k +Ġmit ig +_bl ue +ì ĺ +fade In +ĠUtil ities +ĠM N +; k +< style +- status +ind o +Ġinn ings +Ġg j +Ġ|| = +.e u +: Number +Ġcuis ine +ĠURL s +ie k +Ġw ires +ĉ ps +ie g +.m k +so ap +Ġsom etime +Ġst ap +_s eries +.T arget +æ º +.dest ination +OUN TER +R aises +& A +Ġsmart phones +NI Env +.s dk +Ġhelicopt er +Ġim pe +ĠB irth +A U +b readcrumbs +co ords +Ġexplo red +Ġl od +ĠI p +g able +ian e +Ġart ifacts +Box Layout +ا ر +list ener +.c art +ĠH uff +ĠHind u +ĠData Types +ĠDr upal +IGN ORE +Ġoffset s +ĠR TC +- login +æ ® +ĠQ Object +Ġprosec utor +R ock +_ch at +W ay +ì ² +Ġneg lig +Ġd ude +; < +Ġdeleg ates +_f ailed +/ dev +/ work +( New +et able +() " +( Icons +Ġp ork +ĠModel AndView +ĠV IP +ĠK or +m ix +Ġox id +ĠSC REEN +ĠFour th +/ ",Ċ +Ġte e +ĠSte vens +t icks +Ġp ledge +ib bon +ĠLo an +Ġne o +n umpy +ĠShared Preferences +- oriented +ĠLogger Factory +ĠGraph QL +zen ia +" _ +W omen +.c ast +Ġdeliber ately ++ b +ĠAr n +font Size +Ġm aze +Ġbl amed +.m as +} )čĊ +eler ik +Ġsc anning +ĠWork shop +Ġfind en +Ġca ut +UI Font +( return +al in +cast le +//////////////////////////////////////////////////////////////// //////// +Ġincent ive +op ath +b lob +Ġcigaret te +Ġfert il +*/ ĊĊĊ +ĠSh ar +Ċ ĠĠĠĠĠĠĊ +Ġunc ertain +ĠS ton +Oper ations +ĠSp encer +Ġdef in +ĠS olo +on est +·» åĬł +Ġu omo +G ive +Ġdent ro +; padding +ent ai +ĠC ars +Ġenthus iasm +ĠOper ating +S kip +par ation +Ġprotect s +Ġre ver +d g +ĠC incinnati +Ġconsect etur +Ġm uss +employ ed +a uses +ink le +. Values +£ ¼ +lo v +_W ARN +Ġbook mark +ĠAp ollo +. axis +Ġm ét +Ġop ener +Ġtum or +d an +Ġelement ary +Ġsk ipped +ĠK er +as ia +_res p +Ġdem ol +ĠCan adians +Ġt astes +U Integer +Ġ' ${ +.aw s +RO ID +ri ans +M Q +ord able +Ġcous in +Prop agation +(S ession +ph alt +UL D +ĠSc alar +Ġblo ody +Ġ ঠ+.m ask +, q +ĠUn its +Ġcent res +ĠPr im +. ]ĊĊ +ĠSh aw +P rom +ĠTh ought +Check er +_output s +( chan +E INVAL +Ġb ob +_c mp +P ed +Ġmat rices +Ġvrou wen +Ġgenu inely +high light +(d isplay +) != +Ġdel icate +ĠL uther +ĠM iles +Ġuser ID +% = +ate urs +_B UF +---- ---Ċ +imit ives +Ġsh elves +sl ow +_in formation +LE G +W r +.form s +cel and +/ un +: & +.âĢĻ ĊĊ +=" % +Ġpro st +Ġfont size +uc ión +get ic +am t +=" . +Dec or +B rit +Ġ"" ). +Ġfound ing +.File Name +ĠT ier +Ġdisc lose +á m +.s yn +.View Holder +lic ant +_st age +Mon day +Ġdes erialize +t alk +Ġtradition ally +æĢ ģ +Ø ® +LE X +Ġe h +ĉ ROM +Ġ{ })Ċ +Quest ions +nc py +Ġfix ing +к Ñĥ +_ Key +: x +ĠSTR ING +ĠÑĦ ай +ĉ left +ĠBen ch +ell ij +UR RED +ĠDi agram +} catch +/ time +ĠMiss ing +db name +Ġs ore +ĠW alt +ugg ing +rep resent +ĠG S +ne ys +ĉ page +Ġvol can +(b tn +Ġexceed s +Ġ erg +Ġpil ots +ĠS ed +ers ions +Ġpat ron +R V +/ top +. asset +_c ross +. Editor +.t b +Ġwel coming +SC REEN +) findViewById +C oder + ",Ċ +_P in +ues e +Ġover rides +_ ready +Adv anced +Ġop i +-c art +("/ ", +ĠDe b +CR Y +ĠVert ical +ĠO VER +ĠCorpor ate +Ġ"" ; +Ġste pping +e j +Ġaccus ations +Ġor az +_t ail +Ġindu ced +Ġel astic +Ġbl own +, // +Ġbackground s +âĢĻ une +-s dk +Ġset Interval +Ġincent ives +Ġveget able +_ On +exp anded +p ix +_sh ader +ĠSP DX +@ example +ĠW rapper +.Z ero +Pos itive +Ġsp inner +Ġinvent ed +ĠG ates +оÑĤ оÑĢ +Ġcompar isons +è · +.pr imary +data Provider +add itional +ĉ options +s napshot +.set Horizontal +Ġ" {} +ĠFish er +hal ten +< Type +Ġmax Length +ĠM t +Ġê° Ģ +.jet brains +Ġident ifies +Ġflow ing +ĠDisc ussion +ats by +Ġsch w +ught y +Ġr ivers +.un ique +_PH Y +ed ral +( ll +Ġcs rf +pp ers +ü l +ĠEs pecially +port ed +ĠHarr ison +****** */Ċ +Text Color +ìĬ µ +w ire +Ġstatus Code +ĠFin ish +c ence +ĠMcC ain +ĠW or +( await +Ġ) -> +ĠRegister ed +IN ED +k al +par ison +Ġobj eto +V i +mand a +Ġrenew ed +ĠS of +ess el +.nd array +Ġcr ap +ç® ¡ +.ab spath +( up +Ġclear ance +ĠT W +_C OPY +ĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +Ġforest s +Ġarg uably +ĠA SS +he y +am el +_f ore +ĠSou theast +Ġab used +Ġpract icing +aked irs +ä¸ » +_res ources +Ġp ond +.F ixed +Last Error +ĠPsych ology +Ġ" // +! : +Re usable +Ġmens aje +Ġro spy +Ġb our +Ġvar ieties +Ġem path +(( { +_ org +ĠM es +ĠMag ento +IST ORY +Un less +Ġh j +ĠD uty +J un +, size +Ġpaint ings +Ġdisp ens +d art +Ġbehavior al +Ġr pc +cal culate +fr uit +_m m +ĉp thread +Max Length +Ġc urrencies +_cap acity +ĠO z +Ġfire arm +Ġcoeff icient +Ġbankrupt cy +w art +Ġfat igue +AV A +Ġes pa +_p c +ĠQu otes +_L IGHT +ĠT ickets +Ġrel ates +Ġpublish ers +Ġunlock ed +Ġ// ---------------------------------------------------------------- +ĠInterrupt edException +Ġout look +r n +Ġreb els +W ritten +Ġas ian +ot to +Ġ ĉĉĉĉ +_g pu +T xt +.Image View +Ġsu is +_t ables +.Rec yclerView +Ġwhat soever +è ģ +] ++;Ċ +assert True +_ verify +ĠR ivers +Ġ ][ +J et +id ian +S ibling +Ġgen res +.A ccess +OP S +Ġtr ivial +ภª +al en +в ед +ĠS word +Ġscrut iny +(c b +Ġcomm erce +Ġguarante es +_ad v +ĠL ET +rec io +Ġh ilar +Ġback yard +ãĢ ı +Ġillustr ated +/v endor +. Util +Ġw ow +LO Y +ĠMar shal +"> '.$ +ĠB ak +Ġmod ifiers +d ictionary +ĠSt re +m ultiple +")) , +ĠC ort +'] "). +( admin +ĠCre ator +Int ernet +( ms +log y +DECL ARE +ĠMarc us +<< << +ãģ ł +_m y +(in st +Ġsc iences +ND ER +. enter +Ġit u +Ġbeh ave +P an +omb ies +=' < +')) ;čĊ +ĠM ENU +ĠWork ers +.No Error +Ġbind ings +Ġdis abilities +{ \ +ĠM unicip +Ġco res +ur ple +ĠN okia +us ions +ĠF itness +.handle Change +Ġjav ascript +ìļ Ķ +( dec +Ġpack ing +-de pend +Ġtrans cript +z eros +_ alert +? ",Ċ +lib s +± оÑĤ +Ġ| ĊĊ +tr ained +ĠG ent +ĠR ab +x p +_config uration +å¤ © +_ accept +.rec yclerview +: url +ĠMu hammad +Ġprivile ges +_b ank +uk u +w allet +ĠRO OT +Ġenc uent +? family +ĉ position +Ġc g +Ġprec ip +method s +_f ast +in crement +ĠT iger +_OCC URRED +qu ip +ĠH AS +_d om +Ġw reck +b j +Ġd ern +Ġorg ans +. entries +Ġ_ (' +ram ento +ĠJam ie +Ġp unk +IP P +Ġprogram a +Ġatt ain +Ġpro ves +/s ign +Ġanswer ing +Ġl adder +************************ **** +ĠW almart +ĠCONT ENT +duct or +Ġver bal +ĠP ID +c rypto +_CALL BACK +Ġ= ================================ +Ġpot ent +Ġshort s +.U ri +.un iform +; border +ĠW er +Ġhere in +ll a +ĠI hr +P ixmap +l iteral +! )ĊĊ +g eneric +r ust +_script s +ost o +it us +ĠCoal ition +Ġrem ot +de ploy +ĠEag le +ãĢģ ãĢĮ +Ġimportant e +ĉ object +Ġseason al +ne j +aid u +Bind View +ĠSi erra +-b g +Ġmake Styles +[ offset +G ames +Ġhorm one +AR IO +head s +( select +ĠStart ed +@ param +_de cl +_b log +Ġa ño +\ Api +ĠMil waukee +Pro vid +An imated +Ġcool er +ĠSe ed +. Edit +Ï Ħ +ĠT aking +Ġborder Color +-found er +.Logger Factory +Ġ"" ĊĊ +AL T +ĠL ate +EDI ATE +Ġ);ĊĊ Ċ +af a +Ġcancell ation +At om +ĠB irmingham +emp resa +HE MA +asc al +Ġup side +.V ersion +ĠF older +ĠE ight +ĠV intage +ĠApp Delegate +ĠPre vention +.se parator +ST M +( room +gener ator +Ġc attle +ĉ Z +ĠPart icle +' };Ċ +Ġneighb ours +ĠState less +Ġalt itude +Ġsa int +об ав +Ġconv inc +ĠCont ents +Ġje une +(t s +Serial ization +(c ollection +ĠJ azz +ĠD od +ĠR och +ac io +comm ended +DEF INE +.on load +Ġspecial ty +PL ACE +_MO VE +Ġaccount able +Re uters +Ġf icken +Ġde pr +W ow +V oid +.s pace +à¸ Ĺ +Ġt q +ĠP ets +< $ +(C urrent +ber ries +plan ation +Ġlist Of +ĠTh u +ĠPR INT +Ġm ismo +Ġdo i +ch k +ĠUn icode +( role +Ġvir gin +< Point +_RESP ONSE +-h ouse +ĠVenez uela +EM AIL +Ġp úb +_ex ist +B all +.C L +re ferences +ĠBeautiful Soup +ĉ Expect +TH IS +Ñĥ д +b ane +Ġtemp oral +ER IC +et as +Ġrefresh ing +Ġsec ular +@ synthesize +ac cur +Ġn ella +ĠS OL +.p ipe +Ch annels +èĩ ª +Ġinsert ion +á» ĭ +el ia +Ġadjust able +Can ada +ĠI TEM +Ġcur ves +ĠChe ap +let ing +Ġoptim istic +al lo +Ġpolit ician +_down load += edge +ORT H +Ġmodel o +art o +. rotate +Ġs elenium +æĪ ij +_al ias +Ġrenown ed +.' . +Ġc zy +Ġal les +.Com piler +ĠB ass +Conn ector +.R ole +L INK +Ġc riterion +lem etry +Success fully +/p ng +Ġey eb +asp berry +( gr +Ġd angers +Ġcorrect ed +Ġgl ow +Ġelabor ate +ĠB ears +aw ai +=" '+ +Ġpromot ions +Ġmathematic al +Ġ" ` +_Generic Class +ĠChe f +.S ort +table Name +R IC +Ġvolunt ary +ĠBl ade +-e lect +ĠCom bat +ĠAb ility +Ġab dom +Ġd uck +T mp +åħ ¨ +Ġer ase +.P h +ĠDefault s +p artment +_US B +ê te +; ' +Ġp ads +ĠOb amacare +.T otal +Ġdiv ert +Ġcr icket +Ġrecre ational +( red +ĠC le +R U +Ġmist aken +ĠMont ana +Ġstr ive +_sl ider +ĠPl astic +Ġdecor ated +ĠV P +lic o +ĉf alse +Ġpre fs +( \" +_f alse +i endo +Ġ@ $ +B ucket +act ical +ĠZ hang +.c ols +.B inding +Ġw ax +_ST ORAGE +Ġlaw n +Ġr f +.Sc ene +ĠCal culator +.d esign +Ġres il +л ем +E mploy +ĠPr ices +ĠP WM +ag i +.e valuate +ĉ param +Ġbr ass +bb en +Ġinflamm ation +ull ivan +Ġan not +Ġp H +iam eter +ĠB TC +( box +Story board +Ġcl ay +.assert Raises +| string +.App ly +Ġmatch er +und ed +Ġsatisf ying +Ġìł ķ +Render ing +_app ro +ind rome +AN EL +_f ix +br ush +.M atch +Ġsm iling +on aut +S unday +Ġdelet ion +Ġencour ages +P ull +Ġreven ge +Ġqu arry +tr ade +Ġc ables +(d elta +ites pace +Ġf h +.b unifu +Ġvi el +_IN CLUDED +ĠT ail +ad ar +of s +Ġmet als +g om +_method s +Ġn j +.St d +(w in +$ (' +Ġt urtle +ur on +Ġen rolled +ĠH z +ĠBox Decoration +Ġp ont +rel ationship +B i +³ » +Ġmas cul +Ġsh ades +Ġv r +ĠLog ic +Ġa in +ĠD IST +Ġcoll ar +" profile +Generated Value +ĠP ossible +Ġe ines +ĥ ģ +.time out +ĠE c +Ġjer sey +.D ouble +Ġqual ifying +v or +CRE EN +_A pp +_rec v +Ġali ens +It s +E sc +i ator +ĠE clipse +Ġg h +V ict +ĉ html +to o +. const +Ġant erior +ĠW u +(key s +Ġul tr +_p oly +ĠT ap +ĠB ud +A WS +Ġcrash es +_t ot +Cont in +-h anded +alth ough +ภļ +ific ent +Ġde ve +ut ory +ĠW orth +_M S +Ġfloor ing +Ġsell ers +ĠThank sgiving +Ġp ng +Ġval ores +Ġslee ve +Ġfil le +Ð IJ +Ġappoint ments +Ġv im +User Info +BO OST +Ġpos ed +initial ized +.product s +ĠLeaders hip +man uel +' % +em arks +Per centage +(d ist +. avatar +(h Object +ä» Ĭ +_ iff +ic one +; ) +_n il +Ġab ol +е ÑģÑĤ +Ġven ues +.Con vert +! ')Ċ +.B itmap +sk in +_C OLUMN +Re v +G RESS +g ow +Ġw ished +tract s +.assert False +Ġscreens hot +Ġfo is +Com b +Line Width +ĠGr ab +Ġint ensive +ĉ sh ++ ) +.first Name +_PRO CESS +Ġt ilt +it ored +.L OG +Ġb ak +Ġintention ally +.play ers +(c anvas +)) )čĊ +.Pro vider +_P UBLIC +T alk +ĠL iv +ched ulers +Ġl c +ad ic +feature d +.res ources +Full Name +Ġmean while +B uffers +Ġres olver +ĠS AP +_T E +G NU +ĠForms Module +_ wh +ĠS we +.widget s +Ġcabin ets +Ġsus cept +ĠB ott +activ ex +av ar +ant ics +Ġ" =" +_k wargs +Ġgame Object +ĠAng le +.I ter +mar sh +ĠB irthday +ĠC MS +request s +ĠPear l +_E OL +Ġlin ux +( org +_M ouse +.con structor +Ġz d +Ġk icks +art isan +Ġe ax +K n +pon ge +ĠFin land +Ġmet res +ĠAss essment +part ner +/ pre +! ',Ċ +[ Int +Ġos lo +date picker +/ String +op lay +ĠHe brew +, double +Ġtrab al ++" \ +ĉ EIF +/ text +_F IRST +ĠP ete +Ġe go +Ġextr as +P DO +Ġreg ulate +ĠQ Widget +st s +ĠSh ows +ĠN HS +.c ourse +p thread +ĠF uel +.t imes +Ġ ° +Ġstr ides +($ ('# +( words +Ġrhyth m +Ġsp ont +Ġsens ation +Ġsp ike +C losing +页 éĿ¢ +N umeric +Ġbreat he +Ġfin ale +_F ACT +in ion +Ġch ill +Ġform ally +ANG ED +Ġ' :' +ĠпÑĢ и +a q +ĠFab ric +(l at +ĠPr incipal +Ġer ro +oc ale +N om +Ġf ost +_C USTOM +.int ellij +ert ools +Ġcl asse +adi ents +Ġfundra ising +EN E +_OPTION S +_ ob +// }Ċ +Ġprote ctions +.se ed +N V +term inal +;; ; +P redicate +Ġì ¶ +Ġbomb ing +G F +Ġch ew +)) ). +qual ified +] ={ +list en +C ENT +d igest +E ast +Ġd iver +Ġend points +Ġe e +Ġcolle ague +Ġdissert ation +_com mit +_D AT +. rc +Ġbre asts +ĠR ug +ĠP il +Contract s +ĠBry an +Web View +Ġconcent rate +ĠIn ner +Ġ' | +std out +_S ub +> -->Ċ +V ol +ĠS SD +)) ), +. Optional +Ġnurs es +Ġor b +_ pe +);čĊ čĊčĊ +pl aced +ess er +Ġther apeutic +Ġwhites pace +Ġa ston +Success ful +Ġpr aised +ĠW es +Ġe ighth +ir al +Ġvrou w +Ġf action +_b ias +Ġw itch +Ġnp c +(s b +ĠRod rig +_b ig +Dep endency +ĠAb raham +ard i +C AR +n os +Ġabund ance +Ġnut rients +in stein +.V ert +ĠI SS +< U +Ġsum s +_h ist +Ġfar mer +ĠA br +Sh ot +ĠBad Request +Ġh ass +ĠR ails +Ġaffili ated +æĿ ¥ +Ġer f +IN F +ĠView Holder +min i +ĠR oth +Ġfaith ful +ĠPhill ips +AND OM +]. [ +_P AY +ĠAr ctic +f aker +D igit +M ale +std err +se ys +Ġ Å¡ +_rem ote +li que +Ġin def +ĠIndust ries +it ra +_p airs +< iostream +Ġsal aries +ik en +.F rame +PL IC +_S PEC +ĠMed iterr +Ġsystem atic +Ġinter rog +Icon Button +se a +int ro +ĠIss ues +enc rypted +Ġintern ationally +Ġsn printf +Ġpast a +ĠBrad ley +_ Status +AL K +_P AD +.l aunch +< select +Ġhar dest +Ġph y +Ġ(( * +-s lide +ĠNob ody +S u +Ġas ÃŃ +close st +_initial izer +Ġsupport er +-g en +Ġt ales +Ġcor p +_f u +s at +ne ighbor +.M igrations +Ġal gun +Ġsin on +.S pec +? ,Ċ +.G L +m ale +Ġmon itors +yl an +-L icense +.m atches +ĠA BS +ĠM ast +ĠW allet +($ ("# +Dir ty +Ġco pe +Ġinterpol ation +ous ed +ĠJ ets +.F LAG +.C ancel +.Event s +ne ver +ĠM Hz +> D +Ġs ervlet +bast ian +Ġ> & +S ID +_cl k +Ġdiv isions +} ',Ċ +Ġd ildo +Ġpar ade +m ajor +Ġab oard +; ++ +Ġf usion +"}, {" +ĠDialog Result +ĉ arr +- em +_n r +(h andler +.N ET +.Xtra Reports +ĠSh ah +ĠB rief +- , +Ġprec io +ĉĉĉ ĠĠĠĠĠĠ +Ġt ant +ĠGrand e +/ xml +_IC ON +ĠR etro +un que +Ġn ag +to Fixed +X L +Ġdecl aring +ĠCon crete +ĠAm azing +ĉprint k +Ġdeb ates +D ATED +Ġaest hetic +emet ery +Routing Module +ĠNash ville +W AYS +Ġw olf +Ġobserv ers +OT A +ans on +Ġe a +Ġgreen house +ĵį ä½ľ +Ġst air +Ġimmigr ant +_app ly +pe are +ĠBloom berg +_PL AYER +Res p +æŃ £ +Cho oser +ĠI Collection +P eter +Er ro +.detect Changes +Map s +Ġs queeze +ĠHom es +weg ian +Ġformat ting +Ġnegot iate +ul d +ĠN ep +ĠQ B +Ġeconom ies +Ġ*/ , +Ġredu nd +ĠA ber +.IsNullOr WhiteSpace +yc led +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĊ +_S h +Ġske pt +Ġre created +Ġget Type +Ġmarg ins +Ġcolon ial +ch arts +// @ +Ġprocess ors +è¯ ´ +b atis +æĦ ı +ator io +mention ed +P atient +Ġpre y +Check box +_x path +.s kip +ĠMorm on +ĠMemory Stream +CRE MENT +Ġk u +m eld +\ Data +ĠK ernel +il tr +éĢ ģ +( profile +Car bon +RO LE +( pl +] *( +.m emory +Ġmed al +Ġadvis or +it ät +Ġh dr +ier ung +ĠProvid es +( alpha +Ġteen agers +- parser +.L atLng +] ()Ċ +Ġfel ony +ĉĉĉĊ ĉĉĉĊ +BO OK +Ġsl ash +Ġclear fix +ĠPro phet +å® ¹ +right ness +-f i +.k ind +ert on +J im +Ġmanip ulate +Ġworks heet +ol in +st ars +Ġart ifact +_EM PTY +ĉm ain +------------- ' ; +Ġexpress ing +ĠI Q +ĠF act +/************************************************************************ *******Ċ +_m ass +)) : +Ġcon dom +Ġcreate State +omet own +Ġir r +Ġ> ( +> B +iter ation +ãĥ ª +Ġshirt s +ount y +-> $ +_S IGN +ĠD ale +Ġj j +E asy +F re +ĠN y +Ġch lor +match ed +ĠG erm +- UA +ĠN athan +educ ation +-y ard +- che +h ouses +r itional +Ġprox imity +Ġdies em +áºŃ p +Ġd rought +.a udio +ĠLe o +Ġfavor able +in ch +ĠD aw +rib ly +_st udent +id able +O VE +Ġlack s +ounc ing +.b usiness +Ġre open +may be +_G LOBAL +Ġdress es +ĠEd wards +ens ible +ĠHard ware +ĠEx cellent +ĠTime Unit +CTION S +Ġsched ules +Ġseg ue +Op ens +am men +- Identifier +Ġst aring +Ġhapp ily +ĠH ob +' _ +Ġ" ); +ament os +et ched +Ġ/> }Ċ +. Users +Ġinterrupt ed +Contact s +Ġreg istro +in burgh +CH A +_ imp +ph is +s ay +Ġretail er +.N ODE +/ maps +_L AST +ĠCh arge +_g uard +Coll ider +ĠStateless Widget +": [" +(" ../../ +iox ide +ĠS und +Ġ'' ; +un set +add Widget +л Ñİ +el les +alk er +A rc +Ġded uct +G UILayout +ĠV illa +Ġfor bidden +_ where +Ġ\ / +ĠT ib +_A X +] čĊčĊ +ĠB ir +Ġb end +ĠMA KE +ĠM ET +Ġfut ures +Ġweight ed +"" "čĊ +Ġauthor ize +(pro gram +}, {" +Ġcoeff icients +ê s +Per Page +ĠBath room +ĠPublish ing +G PL +Ġsub missions +ĠNUM BER +j Äħ +Ġaddition ally +em pre +ĠSh el +ot yp +S olution +Ġth under +_ ec +ĠĊ ĠĠĠĠĊ +ĠF ellow +Ġk ay +Ġnew State +ONT AL +Im plementation +.L ook +Ġ ents +Ġl ors +ĠB IG +f ab +Ġaver aged +ĠFe edback +ĠW ells +Ġm artial +Ġind ul +ĠComm unist +ĠFore x +ĠAgricult ure +" [ +Ġqu ar +ĠK ont +ĉ view +. Bytes +des ktop +ĠM akes +akes peare +.Null able +Ġspot light +V B +ow y +(t orch +tr idge +_b ounds +Ġapolog ize +.add Item +ant d +* );Ċ +, u +(g en +ç» ĵ +re ator +ĠC ord +ou pper +.m etro +Ġ ew +ĠW ORD +.A fter +Ġdet ained +ĠHam mer +ex isting +Ġo st +Ġmon ument +-c ustom +User ID +ĠN om +Ġre jection +(d im +Ġsingle ton +ĉd ie +ari ance +re ports +] != +eld a +Ġpreval ence +_reg s +." . +Ġfemin ist +Code c +Ġ **Ċ +(label s +_M ARK +FA ILED +Ġadminister ed +W N +ĠĠĠĠĠĠĠĠ ĉĉ +Ġn oun +w ig +Ġg otta +Ġr if +- im +ĠPaul o +ĠCommand Type +] ))ĊĊ +-z ero +Tr aining +Ġl ord +_ art +re ddit +C ert +Ġpes o +R ot +Ġend anger +.d r +user Info +un ts +n v +ĠTrail er +-f irst +(m ake +Ġbenef ici +-bl ack +i ÃŁ +Ġund oubtedly +Ġm ex +ĠAnc ient +( as +Ġdes cent +P ick +Ġrep lica +$ obj +ä hr +Ġar rows +ft y +ĠLib ya +ug a +charg ed +T ur +Ġh omic +iss en +ĠF ake +Ġbe ers +Ġsc attered +( Time +UT IL +Ġbureauc r +/pl ain +Ġstick ing +FA IL +ĠC ovid +Th ird +_p resent +ĠPier re +Ġë ª +Ġ[... ]ĊĊ +Pro b +ĠTra ffic +ica o +do ctor +Ġ), ĊĊ +T abs +al u +ï¼ļ âĢľ +Ġinher ent +_N o +rit is +ĠPro of +.b asename +ä¼ ļ +Ġch im +ĠProt ected +c rit +Ġpr one +Ġк он +ĠHero es +Ġan xious +Ġan os +Ġweek ends +Ġs ext +Ġredu cer += UTF +h alf +ĠS aw +.m m +Ġnue va +.current Target +.l ua +_EXT ENSION +ĉ reg +ĠC trl +_ align +accept able +Ġrush ing +fr ac +Ġbo asts +F ive + ± +ĠTem perature +> ): +Ġchar ter +RE ATED +Ġsubject ed +Ġop c +health y +使 çĶ¨ +ĠScient ific +Ġfra u +ri ages +ภĶ +.in ventory +ation ale +M ad +min utes +>> ();Ċ +ĠEn v +Ġrecord ings +Ġsusp icion +sql ite +ĉ read +ãģ ¦ +Ġwor ries +.put String +ĠSh anghai +( uid +r er +ĠvÃŃ de +") : +Ġmethod ology +Ġк оÑĤоÑĢ +cc c +av ad +Ġindu ction +ĉ Thread +, string +ạ i +neh men +u ition +Ġ* __ +.em f +Ġì ľ +/th emes +ĠN ine +. One +ĠEm bed +Ġf az +u ations +Ġpriv ately +Ġl ing +[ F +ush i +Ġlaunch es +( KEY +G MT +Ġaim ing +pat ible +ĠB iden +i w +ĠD egree +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġ$ ('< +á rios +to UpperCase +ìł ľ +ĠE UR +Ġovers ight +Ġtable sp +Up dates +.m akedirs +Ġhum idity +/ template +Al ways +( IS +_c ert +D ig +Ġunder way +ort on +ĠHur ricane +Ġsp ends +ĠSeg ment +Ġfl ies +ĠT oggle +ĠLyn ch +Ġs enses +ĠK os +set Enabled +ist ically +Ġtest er +Ġadministr ators +Ġtag ged +Ð ĵ +Ġshort cut +ĠRes olution +Ġsuperv ision +ĠAsh ley +Tr acking +ul atory +and el +ist en +Ġun re +(d iff +ANT S +Ġr ider +Ġs Äħ +.S eries +_ orders +ORIZ ONTAL +Ġret ention +ãĢĤ čĊčĊ +Ġdi agonal +ĠC ancellationToken +_ Internal +Ġru in +.Q t +ocr atic +T el +ĠAn swers +m atic +Ġx p +at em +_j obs +_ any +Ġsen iors +Ġland mark +ĠQ List +Ġman eu +ot ify +/ ";Ċ +/ server +ĠPhil osoph +uten ant +( io +h z +Ġauthentic ated +d v +- Compatible +Origin ally +, function +ãĢĤ čĊ +ĠRepresent ative +as ily +irc uit +.d t +(m ath +.M arshal +[ , +ĠC ities +_ turn +| )Ċ +Ġcant idad +al ter +ĉ ui +ĠNe braska +Ġsk irt +.b g +Shared Preferences +( style +Ġg rief +g ew +Ġsaf eg +ol ang +_l ists +ì Ľ +Ġgran ite +Ġhott est +.j dbc +.C ustomer +Ġâī ¤ +Ġwa ar +_sc ene ++' / +ĠJ TextField +Ġse ating +Ġwe ars +Ġ` / +C ases +ĠY outube +ı m +Ġbal con +, G +Meta Data +- price +SC R +Un ity +Ġtr unk +={` ${ +Ġearthqu ake +Part ial +Ġsub st +Ġelim in +=" '. +//* [@ +Ġsuperv isor +vro let +_ article +Ġp ane +b io +Ġmot ors +N M +F rank +Ġon ion +- word +Item ClickListener +Ġb rit +end encies +Com puter +_r unning +( day +- he +(n amed +ĠS ach +о Ñĩ +c ampaign +.Ab stract +(w rapper +.p ay +Ġu w +Ge o +r ails +/ select +icht e +son s +E VENT +Ġal iment +Pro viders +A wait +_INTER VAL +. off +Ġgl uten +_cl oud +Ġw en +.ex tract +ĉ button +/ MM +Part y +Ġdem ographic +_err no +Ġh iking +(' ')Ċ +", @" +Ġw it +r á +olog ie +ĠSt yles +ĠBrowser Module +.Request Mapping +ic ans +P AGE +cre ation +ĠF erguson +ud ed +num bers +ĠGT K +Ġpresent ations +ĠB obby +_s pan +est yle +Ġilleg ally +abel a +Ġbattle field +cap acity +ter ror +] ");Ċ +Ġwar rior +le ader +ĠDB G +ĠRe venue +Ġvig il +Ġcounter parts +( Error +ACT ER +Ġhe eft +Ġselection s +ze ug +t om +-t wo +. ;Ċ +_st atement +ĠA id +ĠV ul +_r gb +Ġpr izes +Ġedit able +ĉ form +ın ı +.de cor +D emo +lic es +Ġen ctype +rat ulations +ĠR OS +_ch ars +ĠJ ahr +part ial +Ñĥ ÑĤ +ĠRe ceive +ĠL ands +AP TER +Ġch opped +.. " +ĠAn aly +ĠU ID +ĠR adeon +ĠB ee +Ġun m +> M +.find all +Token izer +ĠWH AT +Ġs j +D rawing +E ss +ON D +Ĭ ¶ +(p acket +âĢĶ but +Inv ocation +ĠN uclear +? ;Ċ +Ġgrand es +ĠC rypt +rem ark +Ġ'../../ ../../ +Ġin ability +m agic +c ats +Ġsim ulate +: ${ +in flate +Ġen er +: NO +ip les +Ġmer it +ĠR ated +Ġgl ue +/b log +Ġg ren +Ġthr illed +.C H +unc an +ĠPR IMARY +Ġper sec +Ġfe ared +.M IN +ĠThe ater +é Ĵ +ategor ie +æ® µ +Ġappet ite +s quare +ĠAlex and +.User Id +_g t +_ enter +Ġgradu ates +Fragment Manager +Author ize +-N LS +(M y +Ġtri umph +ust ing +_PARAM S +Char acters +(: ,:, +_B UILD +M Hz +Ġwash ed +Ġun cle +Ste ve +ard own + ${ +_confirm ation +Ġtro phy +Work s +ĠElect ronics +ĠMediterr anean +_m etrics +Ġannounc ing +ĠD AY +_pro to +Ġp ear +base Url +ĉĉĉĉĉĉĉĉ Ċ +Ġcoord ination +: N +.an imate +ĠC otton +_h it +â ľ +Ġjet zt +if ter +(f ields +own load +ific acion +.c uda +ĠLi u +> equals +ĠA ce +ÑĢаР¼ +ĠSuper man +ĠGarc ia +Ġarrest s +ag ar +Ġ{} ) +Ġmac ros +rou pe +ê tre +Ġtw isted +str uments +_ (" +_ vertices +ĠTrans ition +и к +[ max +m ind +Ġaccess Token +Ġun le +m us +c op +ĠF actor +Ġcon ced +Ġre tr +.l inalg +-s lider +ob l +_Static Fields +Ġz ombie +s elling +Ġch ap +Ġsh aking +ĠTrans late +ĠAm sterdam +ĠE TH +_EX TERN +k d +_d isc +Ġpreced ing +Ġpri x +Object Name +_mod ified +ard ware +Ġ?> "> +ĠD W +` ${ +Ġ?> ">ĊĊ +Ġspin ning +_p ending +Match ers +. Keys +ĠP V +en us +ant is +Ġdisc ard +Ġh aul +Ġem pir +Ġpath way +Ġo ak +м ен +-ind uced +Ġimp air +ĠCal gary +.is Hidden +d z +_ include +Ġg m +Ġ' (' +P Y +uggest ions +Ġcommod ity +c ro +/ sub +Ġget Instance +ĠLeg acy +ĠK il +B al +( short +In form ++ x +* r +ĠHope fully +or ate +Ġmach en +Ġtreat y +ĠO ri +.p ublic +-h orizontal +Ġtact ic +Ġb ord +w ares +Ġam mo +ĠL ists +Ġequ ations +/ her +ĠNS W +B ounding +_C ollections +Ġav ail +.Drop Down +è ° +Ġh h +Ġl Ãł +.p b +Ġmemor ial +ĠAT TR +Ġexhaust ed +Ġt sp +ĉ redirect +Ġlik ewise +ST ER +L java +Ġcondem ned +oca ust +(str ict +Ġexem pt +Ġs ms +Ġex agger +S YS +Ġl ounge +: ^ +Ġto dd +de b +ator ial +ĠPort er +Ġtu ition +Ġexem pl +Ġp aren +.line To +Ġkid ney +Ġç a +Ġc ui +ï¼Į 请 +X C +Ġmo ż +Ġnomin ated +l ung +Im Gui +ĠB uzz +Ġstere o +port al +res as +Ġk lass +Ġdraft ed +Ġproject ile +/g pl +(param eters +* )Ċ +Ġassist ed +ĠNS Integer +s itemap +:n th +.View s +.Argument Parser +Ġme er +z ier +ĠD ig +Ċ +Ġpl ag +p ine +Ġblank et +Ġ: - +Ġl cd +------------ --- +(" " +Ġtact ical +ĠRon ald +ex tr +ĠF est +Ġf uer +-n avigation +Ġk b +gh ost +Ġhandle Change +_cl s +() != +Com parator +.v m +ĠCo x +_re view +/ @ +_c ookie +Ġrecogn ised +ld ap +Thread s +ĠSex ual +ĠB earing +(S QL +Ġx r +Ġth igh +URL Connection +ĠSU V +Ġm Context +Ġinc idence +ĠE ste +.s up +_t e +(EX IT +C MD +/ "> +Al most +ĠU ne +Ġand eren +ĠSingle ton +Ġb ore +Th ink +Ġn arc +] initWith +_sh op +(str ategy +! ', +her its +ĠDes k +_m achine +.net ty +ı nda += < +ĠQ R +ĠS idebar +.split Container +Ġon Success +Ġmon key +En joy +(n odes +pect rum +Ġ(* ( +ĉU INT +, height +ĠNetwork s +.t ail +.l inspace +Ġ" ... +List en +Æ ¡ +.Ch annel +- defined +Re peat +ad just +ER M +_ application +.assert NotNull +- stream +Ġr abbit +Ġposition ing +Ġw oke +Ġf ing +Ġmulti player +Ġregister ing +un til +Ã¥ n +( :: +uss ions +Ġpot ato +ĠE quals +.S up +/ap ache +Ġ( = +. ") +.p tr +ĠSpe ech +.cl ip +ĠGab riel +Ġmusic ian +/ issues +.sh op +ĠH ier +_RE T +_b ucket +ãĥ ¡ +av s +Ġro z +fl ower +Write Barrier +ĠMil an +Ġlegisl ature +ĠD oll +Ġprov ing +.concat enate +âķ IJ +Ġg char +cdn js +b les +ĠList ing +л о +.xr Label +ĠS ak +just ice +ĠVal entine +un less +Ġp iger +(r un +Ġtest ified +AN A +ĠRem oves +)) ));Ċ +rec ated +ĠRuntime Method +Ġcon qu +ãĤ ¢ +Ġt issues +ail er +ét é +- Star +Ġfl ames +.set Icon +Ġsup ern +Ġvag ina +- variable +Ġwell ness +C UR +Ġbel le +.get Request +Ġp oco +ben h +ag ens +Ġsp ill +ĠJ ur +Ġdispatch er +н ого +emon ic +(dir name +ĠÐ Ķ +Ġpas se +Ġg anz +ric ing +E U +Ġmuj eres +ess en +.at tribute +j j +ĉĉ ĠĊ +[ ^ +Ġstrtol ower +lex er +ect ar +hot el +.s quare +Ġr all +Ġlower ed +hand led +Mark et +ĠUs es +iv as +.B usiness +ãģĹãģ ¦ +D IV +Ġw asted +Ġav oir +ê m +_ACC OUNT +. et +ĉ SDL +k ap +Ġf ox +up pet +{ },Ċ +", ' +F avorite +P END +ĠA ES +} ), +Ġded uction +Ġpol ÃŃt +Ġcomponent Will +ĠT elerik +_SE LF +Ġm use +C raft +Ġd ens +ठ¿ +( tp +Ġt asty +Ġbal ances +Ġded ication +ĠWall ace +Ġun law +\"> \ +Ġm um +- update +ement e +Ġs oda +Re public +as mine +é ric +( Status +ĠJson Convert +ĠD isk +.Red irect +Ġfilm ing +/m ol +R o +Ġv ille +Ġtrab aj +Ġsyn thesis +reg a +Ġr l +S cheduler +ISH ED +current User +(error s +' h +_b ot +x imo +ĠUS ART +_s uper +_DEC REF +н ой +_RO W +Ġprom otes +ĠT A +Ġhor as +ĠRep resents +Ġname of +ĠEx c +ĠGar age +Ġse ine +, # +Ġher b +/ resources +Ġple aded +.r adioButton +Ġæ ĺ +O ps +ĠN est +c string +ĠDef ence +Ġref ere +_le af +Ġrevel ation +ë § +.execute Update +_W ORLD +Ġexp ans +(" \" +j ab +Ġdoub ts +ĠGe ometry +Ġintrodu ces +Ġsen ators +Ġcan al +.h elper +ĠBi ology +_SE NS +.pre vious +-t ouch +ab it +Ġimpact ed +Ġbr ackets +.d irect +acc um +Ġtest osterone +ĉ action +ĠCh ance +Ġpe aks +CppCodeGen WriteBarrier +Ġun belie +_p ress +.R el +ang led +/ templates +-- >čĊ +l ime +Ġsufficient ly +_ nt +Exp and +.is file +Ġis Empty +Ġq t +Ġmul her +ac ob +Ge orge +å¸ ¸ +Ġass im +as o +Ġcompr ised +O V +(CON FIG +ĉw riter +Ġdes p +Ġten ure +(c r +.p ool +ĠB rend +Ġc ensor +(time out +Ġple a +.W rap +Ġtight ly +ĠW ere +ĠI gnore +abe i +Ġbr idges +Ġcondem n +Ġsimp licity +Ġrout inely +Ġblack s +j b +ĠP it +U tf +Ġ/ Ċ +re load +Ġset Object +/g lobal +Ġf atty +Ġsock s +Could n +Ġerot isk +æĿ ¡ +ĠPress ure +ĠM az +n pos +tol ower +ĠE Q +ute ur +ĠM oment +Ġet a +{{ -- +Ġgraph s +ĠGu ar +r ine +( -- +ĠHttp Status +(st udent +* np +Ġrail way +Ġas ynchronous +_v m +'] ,' +, text +mer chant +(G uid +ĠG ra +ix er +fetch All +.add Listener +fl ip +* $ +> (), +Ġsun light +ass igned +Ġab c +ĠC OLUMN +ĠðŁĻĤ ĊĊ +) ... +Ġen semble +Ġnew line +_S INGLE +ied ad +Ġdark er +orm ap +Ġl ion +pl its +Ġillustr ation +ĠI EEE +Ġv ista +ous ands +****** * +ĠTom my +Ġh ue +S el +Ġa ura +ĠTher apy +Ġanim ator +.con straints +Ġv ague +(" ") +Ġvill ain +Ġbless ing +Ġstring Builder +ĠM isc +ĠD IR +f ax +- node +ĠWalk ing +ĠA U +s ess +Ġgr ill +VERT ISE +ĠF oods +Ġt ournaments +à ĵ +ĠMar sh +Ġw onders +Long itude +.Command Text += input +_enc oder +page Size +Ġget State +> >Ċ +.g rey +p od +Ġread ings +Ġre consider +Start up +Ġexc er +.b alance +_c ycle +_T ime +LOC AL +ĠE FI +ĠRe yn +.set Foreground +by n +Ġdis connected +ACT IVE +Ġembed ding +ick ers +Ġsurround ings +* c +Ġgar ant +Ġb f +Ġw ipe +Ġ ä¸ĭ +_T RA +ado x +ç ķ +Ġsu cks +ĠS ongs +ĠAssoci ates +ĠB ald +ĠB rett +ven ile +Ġv t +Ġin ade +Ġres igned +ĠGl enn +.p attern +.Data Bind +Ñĥ м +Layout Inflater +ch et +ĠTest ament +.m s +Ġp av +ĠReact DOM +ur dy +AD ATA +M u +/ actions +ĠJ s +_ex tract +ĠBr ing +: id +str t +iv ation +Ġoutr ight +az u +loy ment +и Ñı +al do +ĠP ublisher +E ducation +Pa lette +_d rv +Ġ($ ( +ĠAnd a +Ġrem edy +Ġincons istent +te ction +Ġregul ators +Ġshort est +(p air +ĠInstall ation +Ġdefend ants +Ġ( ); +-l arge +M el +Ġthreat en +н Ñı +Ġfet ish +ot ine +_d ic +Ġ< $ +Ġst agger +sp i +$ response +S erv +-b orn +j os +ĉ img +ĉW HERE +_l t +å½ ĵ +.c ost +ĠT ue +.label s +ĠL V +wcs store +ĠJes se +ภ« +Tr ade +Ġpredecess or +ë Ĥ +fin ally +_g eneral +ogg ler +_REG ION +n ement +Ġblog ger +ĠHar bor +ĠD ataset +[ w +Ġattend ees +. ico +max imum +.Un lock +_SY NC +ág ina +Ġdown s +ĠW ii +]) / +Ġkick ing +unic ation +ĠD AC +ĠID S +ĠR ental +Ġcurrent Time +Ġvacc ines +ĠDev il +Ġn ors +_m ouse +urre ction +(n o +Ġ> čĊ +Ġaggress ion +Ġbre eding +.s ymbol +im an +Absolute Path +ĠWH O +_fl ush +- root +arn a +& M +Ġf athers +ĠR ocket +ive au +Ġw ander +Ġcom pos +ĠWar rior +ĠSe at +ĠClin ic +_in voice +(dis patch +Product o +at uring +oss ier +ĠM AY +Ġd agger +Ġsanit ized +ĠR FC +Ġpro ph +Ġur ine +Ġgr ind +ĠExp anded +des cripcion +-f w +ĠK erry += name +Ġch k +Ġnation ally +Ġthe e +In c +Ġ? >> +.R adioButton +.Http ServletResponse +/ Y +ĉf ield +Ġhom me +y per +Ph ysical += v +Ġdr iv +ĠErr ors +Ġc Äĥ +De ath +ĠW INDOW +Ġpo et +ĠSh arp +ĠImm utable +ĉ create +Ġge ht +ĠRe form +ais er +ĠInitial ization +Ġimm unity +.com pose +Ġlat ency +ĠLeban on +ĠPar ad +Ġfu els +ĠEx hib +co h +% ">Ċ +ĠCL I +) initWith +-Z a +_C LEAR +reg n +Ġfin ances +.st andard +_C ATEGORY +.lib rary +Ġtravel ers +_w p +ĠE valuation +start ing +Ġ )),Ċ +ep isode +ĠV ariant +Ġda emon +ĠJul ia +ĠN R +Ġdoub les +< v +/r untime +Ġinterpre ter +ĠIN DEX +ĠHol mes +_D IM +Ġp addle +_ex ample +Ġfore ground +.r outes +Ġs owie +S UCCESS +ĠC DC +ĠB D +_ - +as ured +W riting +Ġcurrent Page +( answer +ĠASC II +à ¨ +Ġsocial ly +yy y +ĠSpecial ist +(c ustomer +ist ani +ke st +ĠM ak +Ġth o +. pt +( comment +ĠCon verter +g am +b ins +. tele +ĠVeter ans +_AL LOC +олÑĮзов аÑĤ +inn amon +; width +oh l +Ġfant as +Ġs ung +ĉ K +( Json +Ġneighbour hood +Ġv ow +Ġs ins +on acci +Ġepoch s +im agen +.Ch ange +.my batis +Se ek +W ER +管 çIJĨ +Ġinter ess +_ Event +eder land +Ġterr itor +Ġci udad +uck ed +Ġsn ack +Ġtransport ed +ĠMan ifest +ĠD AT +_th eta +Ġw ont +.ĊĊ ĊĊĊĊĊĊĊĊ +Ĭ¶ æĢģ +ĠEp ic +De ck +l tra +_Z ERO +Ġ[] ; +/ scripts +Ġ---------------------------------------------------------------- ---------------- +æĥ ħ +Ġwe ed +N BC +Ġrap ed +ĠG ateway +[ M +ĠTime out +ench mark +.View Model +Ġporn os +ĠY a +th ritis +ĠFly nn +Ġme ga +ac in +Ġtrib al +.app le +ĠB lo +â n +ib i +ro v +ĠL ives +^ . +get Request +ĠEst ablish +cont ainers +Ġst arring +Ġcele brities +ĠRel ative +ĠHe ights +Ġtq dm +ĠNorth west +iv ic +ĉ cl +Ġautom otive +ent ric +Ġfort unate +Ġfire place +se ud +nick name +; s +_C AL +h alt +(n s +_de leted +Develop ment +m ovies +Ġident ities +Ġprompt ly +ا ÙĨ +Ġant e +Ġ" ',' +åı £ +imp se +Ġy ap +Type Name +Ġb itch +Ġassoci ates +HE ME +- empty +ĠØ ª +ol vers +Ġpist ol +Sc oped +ag ner +'] ==' +ĠI MP +ex c +Ġo mitted +Ġmind set +Ġ[] ( +Ġor n +_C AM +A vg +Localized String +ĠN atur +Ġcom poser +ĠPlay ing +Ġover d +_ utf +.s k +ĠF ol +$ page +, Object +Ġbe es +al ary +bul let +_lib rary +O ffer +loc ated +Ġ(_ , +âĢľ He +ĠOwn ers +) ).Ċ +Ġb ri +.Ad min +kt ion +лÑİ Ñĩ +Ġerot ici +Cancel led +Ġa gr +re views +_d ma +RI CT +Ġg fx +mp i +pp o +Ġ// @ +Ġupper case +Ġcommit ting +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +User Data +Ġv ai +ĉs ort +Ġcongr at +Ġd ioxide +д а +. area +ĠJosh ua +ĠK och +_b reak +az ure +ist ical +_AL PHA +_ views +Ġelim inating +OM B +en umer +ĠHy dro +(* ( +ERT ICAL +Ġinev itably +Ġst ole +-e ast +ier on +Ġl inger +/d oc +Å º +ĠAl ready +as io +Ġ-- Ċ +Ġabb rev +ĠAt om +h im +ĠINS ERT +s un +âĻ ª +CON NECT +er ator +ĠM anning +Ġ: ( +g as +=> ' +Ġquery set +; }čĊ +ĠPop ulation +uted String +res ident +_F ONT +ĠRes pond +Ġobsc ure +Ġo bservable +ĠContrib utors +k on +ĠMus k +ex ao +ĠT ub +Boot Application +S OR +.H orizontal +.find By +.p ower +Ġposit ively +ven ience +ĠJ ong +Ġwh istle +Ġз наÑĩ +Ġl ending +Ġdestruct ive +Ġon Delete +author ization +(); ?> +_ original +sc ience +at ra +?, ?, +ĠAs c +Ġconvinc ing +$ a +org en +_D ate +ĠPro vide +Ġlon ely +) 'Ċ +ex change +; ?>Ċ +.f ast +S amples +L ondon +'] )čĊ +ĠI onic +Ġp esso +ĠKn ights +ĠR af +_attr s +Ġrepe al +> Main +ĠOrder ed +_N ew +=" "> ";Ċ +ĠS ERVER +ĠHE ADER +_ velocity +ĠIn voke +.timestamp s +Ġs ulf +I QUE +Ġinhabit ants +ph ins +azz o +Ġmon o +Leg end +Ġnon ce +IF E +; ";Ċ +- create +" ",Ċ +per mit +ĠImm igration +Ġpath name +ffect ive +âĻĢ âĻĢ +Ġex ams +- event +ĠT ill +[m id +F IX +; color +( Order +_tra its +Ġorder By +Ġs unt +ĠNich olas +Ø ² +Ġsun ny +in ers +Ġaccess ibility +ĠH B +.com p +ĉ op +Ġminor ities +ethe us +Ġcollabor ative +pr it +H IR +Ġwr aps +ĉd raw +g od +ĠI X +.app s +ĠN M +Ġirre levant +ĠT igers +Ġdi ag +G V +ĠAccess ories +k ont +Ġsimpl ify +ĠF avorite +_t ools +([] );Ċ +Ġtow ers +B es +Ġhun ter +Ġsal on +(b uff +ĉ debug +Ġmal ware +M oving +- options +) +' +ĠLO VE +_S OCKET +_f in +ĠDel aware +Ġsher iff +-in valid +ĠF ULL +Ġп од +el as +" strings +ĠRepresent atives +s urface +res olved +ht docs +)) :čĊ +Ġpress ures +Ġnorm s +Ġpl a +Ġs urname +Ġpost al +ĠDep art +Ġsla ughter +or ida +Ġhe bben +Ġdes ar +comp act +_L ANG +åIJ Ī +op oly +_r ad +ĠST DMETHOD +L azy +ĠĠĠ ĉ +... , +( web +ĠP ont +Ġet was +Ġup ward +_h at +Ġ], ĊĊ +Ġbase Url +Ġworry ing +-add on +(get Class +S PI +Ġcapt uring +) },Ċ +Effect s +Ġcompet ent +Ġf oul +Ġsubscri bing +ĠO BJECT +IX EL +b ucks +( edge +(p ass +ĠPet erson +Ġbo obs +ĠD elay +_s quare +el im +ot ers +_P C +% E +on click +ĠSV G +Ġto pped +Ġf ist +sm art +ĠR alph +( owner +j ours +Ġbron ze +ĠArgument Exception +( original +_S CALE +_c p +Ġrecomm ends +.set Style +S ure +L AND +Ġrepe ating +M att +. Visibility +Ġenter prises +.Set up +(sc ene +ĠRe active +ur ge +b w +.P ut +p ersist +.c ookie +ĠAud i +` s +sup plier +( Form + ¡ +_s o +Į Ģ +ĠLeg ion +t te +N d +L oss +( attrs +.sc atter +Ġg room +Ġgl impse +Ġn ails +Ġcum ulative +Ġf azer +_s ervices +.N um +ib ilit +_res olution +ĠT x +umin ium +op a +.s chedule +sm tp +ภķ +ur ry +ü k +go og +_sign ature +.int o +ĠSte ps +Ġhome owners +ĠNS URL +ĠP AC +ĠĠĠĠĠĠĠĠĠĠĠĠ ĊĊ +> ')Ċ +en h +Ġinc ap +$ MESS +Ġmo ins +ĠF i +Ġoff season +press ions +> .Ċ +ĠGr ass +ĠGo al +_p df +Hand lers +Ġstack s +.get FullYear +=[ ];Ċ +è½ ¦ +, V +(s plit +Ñĥн к +Ġbake ca +Ġ~ /. +pe z +t ails +ĠG len +Ġset Image +ĠCom ic +B LOCK +ĉ This +o ader +Ġcapital ist +_ST EP +( Boolean +ĠCor rect +r ina +Ġconc aten +å® ŀ +() :ĊĊ +Ġun anim +ll i +al ars +- ne +Ġdiv or +ĠKick starter +]. _ +< number +/m enu +GR APH +vis itor +Ġimpro per +_N EXT +Ġb isa +background Color +/ input +Ġmo i +Go al +li qu +Ġmiscon duct +Ġcompr ises +aw ns +ĠP ie +ra is +role um +Ġcur se +y u +_p oll +.current User +ES H +]) [ +Ġstory t +)? ;Ċ +* = +ĠB urg +/ layout +_back end +; ?> * '+ +åĿ Ģ +ac ency +( URL +_h alf += l +Ġlist View +( section +.to Array ++ / +ĠRodrig uez +ist ream +Ġelig ibility +:: - +.new Instance +P B +ĠAs sets +ĠCom posite +ĠL abs +ĠHam as +++ );Ċ +Ġbl k +ĠNe o +L uc +@ login +Ġun aware +.m et +_RE LEASE +( ST +AM IL +ri ke +Ġ( ){Ċ +(s printf +ĠAccount s +ĠV IEW +ĠA j +ãĤ ° +Ġwh isk +Ġid i +Ġro de +Ġih n +ĠElement ary +Q ty +Ġintrig uing +Ġå ¤ +J obs +ĉ offset +ĠAh med +ĠTal iban +Ġè İ·åıĸ +Ġinject ed +.Auth entication +_line ar +.Dec imal +Ġapp les +Ġshare holders +Ġb aked +.d iff +ĠE ddie +ok ers +Ġconfront ed +vo ices +Ġt us +ĠSp in +N ODE +_ Un +CT X +/g oogle +Tem perature +Ġ' '). +Ġmagn ificent +Ġstart Index +semb les +Any one +z k +eh en +ĠD ame +. strict +Ġrepl aces +Ġline back +Ġpush es +Ġche ek +ĠSh i +_BY TES +RE A +ả n +_CON NECTION +G ateway +ĠTr avis +ĠA X +ĠBas ically +ĠUp grade +à ª +th emes +erm o +k or +F emale +_att ach +ĠìĤ¬ ìļ© +Ġpo z +============ ==Ċ +(s ymbol +ĠS ector +__ )ĊĊ +_p adding +ï¼ļ " +Ġf abs +Ġr anged +set Name +Ġp error +â Ĺ +ĠFile Reader +Ġful filled +_C urrent +Ġdom inate +Ġsm ugg +Post Mapping +_for ce +Ġb loc +ĠG iant +(v ideo +ĠC U +System Service +Ġ elf +Ġkont akt +ë ª +ke es +gt k +Ġparam Int +Ġmark up +u ales +Ġaccount ed +Ġgang bang +RY PT +ĠW rong +Ġcred ited +ĠM ESSAGE +Ġfl aws +Ġbb w +Ġmetab olic +ĠO EM +/ event +(C ollectors +mont on +ap pear +Ġopt ed +Ġche at +Ġd av +ĠPro ceed +Ġê ¸ +ank ed +и з +ans k +ĠH ang +ĠC ler +Ġdis gu +Ġc map +.cl js +Ġa ument +le z +ĠJo ined +_re ceived +Ġa erial +ot el +Ġgre et +" s +ĠGen esis +ĠCal if +pan ion +Ġtail ored +m apping +and Expect +.tr ack +at omy +ĠO w +ull ah +.Y es +ĠSimple Name +db h +' en +Ġnons ense +Ġphilosoph ical +(get Context +Ġis so +ĠA CE +start Date +Ġb ÄĻd +ĠAUTH OR +ĠGlo be +Ġinsect s +_A l +ush ing +è® ° +/ Home +ĠLocal Date +need ed +hes ive +Ġill usion +äº Į +Ġtr at +x o +/d etail +_M ATCH +Ġbroad band +Ġw al +ĠIllegal StateException +IRE CTION +Ġnor theast +es ium +ĠClient e +ul ance +nt y +Ġt ecn +Dev ices +Ġgr ains +ĠO g +ĠS EL +ud iant +Ġ++ ;Ċ +Ġexplan ations +oc co +Ġdi ets +Ġco hort +( controller +.Iter ator +-r ich +ro cess +G D +Ġcar bohydr +Ġfri ed +ĠEmploy ment +ìŀ ¥ +ĠLeon ard +_ ${ +qu ares +Ġcompan ions +Ġpar is +Ġstim ulation +ĠZ oo +Ġre levance +ĠCol our +Ġspe ar +ot ional +ĠL ite +ĠK osten +Ġà ³ +_att achment +orph ic +Ġdam it +Ġd lg +Ġthr ive +CH ANGE +ĠApp arently +Ġat ual +Ġroot ed +( images +aw i +ari at +Ġch erry +STAT IC +m nt +ĠUser Id +il let +ĠHis panic +Ġn ak +Ġcent ro +Ġdim s +_initial ize +ı k +ĠCent ers +RE N +Ġevolution ary +ĠTop ics +_d amage +em er +Ġr und +Ġpun ished +Ġcub ic +f air +[] ;ĊĊ +Ġinstant iate +Ġover see +- delete +unte er +start Time +ĠP ipeline +_G AME +ĠC ir +ĉ Null +.Format ting +uc umber +ĠR ide +Ġz oo +Ġcheck er +åIJ Į += C +Ġg rit +"); // +_x y +ĠDe claration +Ġcall able +F oo +ĠList Item +Ġin accur +ml in +ĉ Data +Ġev olving +aw an +Ġca fe +fol k +_ID X +ĠAny thing +ĠPalest ine +ĠGrid View +Ġcol ony +ĠGerm ans +( + +.p id +.js x +ĠSuper ior +Christ ian +ĠL ect +ĉ Game +Ġinstrument al +Anim ations +д ал +ĠMos es +ĉĉčĊ ĉĉčĊ +z s +k te +ä¸ ļ +_D IST +bit map +d B +Ġp ersistence +ÑĢ оÑģ +$ l +B ron +Ġ{ | +_ch art +ĠCon sum +Ġh emp +Ġ" ))Ċ +Ġattack ers +Ġknowledge able +Ġc et +Ġvir uses +' I +Ġpitch er +Ġsweep ing += list +apt ops +.de pth +Ġinstruct ed +ĠR us +benh avn +Ġи н +S ports +Ġon set +æĿ ĥ +. RED +_s i +ĠP ST +.on Change +> tag +ĠR oh +_char acter +ĠLaw s +ĠB achelor +_s wap +.re activex +Ġreward ing +Med ium +- [ +ĠRec ently +J oint +part ition +ĠMin utes +Ġind o +Ġabsor bed +ĠG N +_IN D +Ġsab er +Sp awn +output s +ĠJeff rey +Ġmed ieval +h ed +Gu ide +Ġpsy cho +Ġgl am +E lim +äd chen +_pl ain +ĠS au +-f our +Ġanaly zing +QU ERY +Ġtom ato +_button s +V EN +.set Status +. Url ++ ĊĊ +Ġcompl aining +deg ree +conf irmed +Ġsub t +p arsed +Ġtor que +Ġtroub led +ĠT ARGET +Ġtrad emarks +ĠCo ordinate +ĠV iv +Ġ// }ĊĊ +Ġapr ès +.get Position +(Key Code +ĠSil va +Ġmet eor +Ġendorse ment +Over view +ĠP oss +.In ject +Ġeven ly +Ġvisual ization +Ġw char +ĠH DMI +Ġfun ct +ick name +',' ',' +Ġfor wards +Managed Object +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +ĉ server +ĠOut look +ĠChron icle +Ġdub bed +Ġd ok +ĠW ear +.A L +pare n +. Interface +Inter faces +.c od +Ġd ib +.Global ization +ĠAcad emic +Ġass ms +Aut om +Ġl w +ĠN W +Ġ&& čĊ +Ġproble ma +ĠManufact uring +lim its +-m obile +Ġfil me +/ map +Ġdo it +ĠIn k +Ġsu ed +. arr +Ġunder min +ĠPro c +croll View +__ $ +Ġsidew alk +( that +ภ· +[ q +gram mar +Ġt ë +qu ito +Ġspir al +ext ended +Ġf ocal +Ġdig ging +p as +ĠT all +.pro xy +it ures +TR ACT +ĠRe alm +Ġf eder +Ġorient ed +ĠAltern ative +Ġo we +Ġsour ced +ink er +.d et +S ep +ĠQ ui +ĠPal mer +(_ , +s amples +oy er +ull an +que z +Ed ges +Ġsh out +ĠA chie +Ġha ar +_Con struct +Ġprem ature +Ġre vert +'). Ċ +Ġs chn +filter ed +null ptr +S aved +itect ure +CL A +Ġv l +st ell +ĉ Me +ĠL ip +n ational +Ġwh olly +Ġspr ings +.T imer +ĉs rc +els en +åħ ¶ +Ġcommunic ating +ĠQu iz +Ġt eng +Ġge z +ĠOut side +.S ign +(c s +Ġdisput es +ĠWe iss +ann es +> No +ĠB ach +.remove All +re fer +/d ashboard +ĠA jax +Index Changed +ĠWe ak +' "Ċ +Ġs ights +access Token +ĠJ oi +(d omain +ĉc v +Ġcontin uation +Ġpl um +ad ir +.set Message +Ġ ï¼Į +Ġsw allow +ĠL amp +Ġq w +Ġu u +C oin +ub ic +ĠDe als +r ace +Ġdict ator +Ġmem e +turn ed +ĠJul ie +.grid Column +Ġpup py +Ġp am +Ġ) {čĊ +Ġinv iting +Ġf rench +v im +Ġwr apping +Ġ#- }Ċ +([ - +Ear ly +Ġsh iny +.f aces +Ġreb ell +abc def +ä lt +Ġest imation +ph ys +los ures +_RE L +Ġex clusion +ĠSk ype +we ise +-st op +no thing +ĠE gg +is ors +Rich ard +Ġcounsel ing +Ġcomm em +ĠQ MessageBox +ĠSy nd +ĠFro st +ĠCompet ition +ĠAw ake +Ġt ed +ic iones +ĠDev Components +VERTISE MENT +ott i +.run ner +Ġuniqu ely +.fl ag +ĉ rs +_g eneric +Ġ`` `Ċ +ACH INE +Ġme in +( Application +( br +Ġrat ios +: , +ĠXCT est +ustain able +- www +it les +_T EMP +Ġs yst +umeric UpDown +ĉassert True +Ġw f +. peek +ĠBul g +Ġterr ifying +.M ODE +ĠG W +á r +Ġf ic +Ġcommit ments +- tech +ĠL iquid +ope z +z heimer +a ña +-m edia +( animated +_go al +Ġg um +yst one +.S ET +ĠW end +set CellValue +Ġmsg s +c ash +AL LOC +/ aws +Ġmic rowave +.Point er +ĉ Console +_s orted +ĠFil ip +Pro d +Ġ//! < +ing roup +Ġk s +_T RI +Ġteas poon +ĠAT T +Ġrecover ing +ĠG LOBAL +.P ar +Ġ/> ;Ċ +Ġmar ble +ul ators +ĠC ycle +Ġher bs +_m etric +) ! +_C LOCK +_ Button +H arry +è¿ Ľ +Ġstr ains +ĠApp Bar +ĠCh an +/v ideo +Ġb am +.Pro gress +$ f +lem en +Ġir regular +ĠD uncan +ĠM int +-v ideo +ঠ¾ +ó wn +ĠEM PTY +Ġstack ed +ĠH A +_c ut +Ġwhere in +ĠW ays +(count er +è¯ ķ +Form Group +Ġble w +c ourses +Ġproduct os +ry s +ĠRest r +Ġsty ling +> s +Ġp iv +Ġit ertools +get Repository +ĠI k +_dev ices +lay ui +Ġhalf way +Ġfran ç +Ġtun ing +O A +_N ode +ar de +Ġfier ce +lic ted +# čĊ +Ġbreak through +ĠE rik +Ġb ride +Ġ. " +cul us +ins ide +ĠIndian apolis +ĠE E +Ġy og +urre t +.f s +. grad +_c ards +_ac curacy +_ep i +qu eda +/ org +é ªĮ +Ġcom pte +)) [ +Out side +G reater +ĠRender er +. actor +Account s +Id le +_h ours +ern er +Jo ined +Ġmen j +requ ires +ĠO PER +.remove Child +ĉs p +Ġes se +r ift +xF E +ĠSh akespeare +________ ____ +Ġbudget s +Model State +fill able +- component +oc os +ĠBUT TON +/ io +, out +s ms +Th omas +ĠAr med +res ume +Ġrot ating +ĠV ault +Ġse us +. (* +Ġa mino +Ġ[] );ĊĊ +Ġprov oc +no x +.Get Enumerator +==== ===Ċ +æĸ Ļ +_sc roll +Ġfil med +ĠS oci +g ap +g ro +V ote +" But +_R C +An imal + Ģ +ib ile +Ġaw aken +ore st +in ja +ĠI van +( Command +Ġ ***** +Î · +Ġkv inder +/h elpers +_c ases +t g +ìĦ ¸ +Register ed +ĉp ass +_d igits +Ġcont our +Ġinf ants +Ġjust ification +ĠFort unately +Con tr +ĠonCreate View +_S AMPLE +Ġallow Null +Ġn ud +Ġfet ched +_e qu +ĠUn able +=\" " +> {Ċ +Ġcommit tees +ist ema ++ ". +ÃŃ an +m ant +Ġsou theast +ï¼Į Ċ +dialog s +PRO JECT +charg er +- port +(u uid +. export +S ix +ĠR P +P rem +Ġconsc ience +Ġmargin Right +_d istribution +y aml +res izing +D ock +ĠLoc ations +G Y +Se ed +B UFFER +oss ip +ull en +Th ings +- self +.p oll +PL AYER +Ġå ® +G ROUP +ĠA way +Ġg ospel +xf d +M ary +ĠPort able +T URE +Ġutil is +Ġse it +Ġstr and +Ġtrans c +Ġ( ^ +ĠAl fred +.m em +.c ircle +Ġ~ / +for cing +Ġr iot +pro x +TH ON +iz ación +ĠN I +ro st +Ġdis pro +_in stances +ï¼Į âĢľ +ograph er +end as +ĠIsa ac +ĠP ine +/d is +Ġcolor With +iter ate +_str ide +Ġpun to +.Event Args +( center +Ġneighb oring +ĠPr ison +ĠMess enger +Ġepid emic +da o +_com plex +Ġgr avel +_D IP +é ment +ĠA ri +_bit map +.qu it +( valid +Ġp end +Ġrespir atory +Ġre bound +Default Value +ãĥ Ń +Ġcomm its +.test s +_f r +it et +.s f +Ġspace craft +c ritical +Ġde pressed +ĠAny Object +Ġun b +Ġdisc ern +(m ysql +L atin +ĠB og +ĠWild life +To File +iox id +@ RestController +Ġ"$ ( +Ġ<< " +Ġdefect s +Ġdat um +h in +Ġreal izar +any ahu +ĠS ig +@ Data +ad aptive +ĠC atherine +.c r +ĠCO OKIE +Ġp ictured +ĠFight er +Query able +ĠAny way +ĠGL FW +_n amespace +_ ft +Ġ] ) +Organ ization +Ġconstit utes +Ġqu and +(ch unk +"/ >čĊ +ĠL akes +main window +Car thy +sp in +(c sv +: red +-com merce +ภ¹ +Ġdiscover ing +Ġe co +_f ac +inc eton +ĠGre ens +j wt +Ø µ +ĠBron cos +ĠGood s +(G TK +Ġreturn Value +Ġsi empre +Ġneut r +w ent +ĠN atal +Ġenthusi astic +á» į +F N +/d atabase +C atalog +Ġbr un +ĠK ash +_P l +isc rim +, width +Ġin mates +Ass ignment +ĠH aven +Ġplay ground +ex am +@ Controller +ul iar +.get Parent +Ġ" ;ĊĊ +: size +iss ors +Ġf is +Ġal c +ens ation +ĠN ixon +Ġmight y +- str +_s pecial +_A DC +ĠTw ig +um bling +- address +Ġher oin +Y TE +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĊ +F riend +Ġa ve +ĠP NG +ĠKurd ish +DataSet Changed +Ġbl ades +br al +St eam +Ġsig u +IRT UAL +ac os +UD P +(d atabase +he c +ĠString s +_scal ar +ĉd esc +ĠT LS +; "Ċ +ĠCor byn +Simple Name +u ell +ĠEnt re +ell ites +- place +Ġfrank ly +ĠE rf +CE L +Ġpa ÃŃs +Ġh edge +Ġlat ent +ĠIR Q +ĠH erald +ĠP rec +ë³ ´ +.T EXT +Sal ary +Ġaut umn +Ġtrav ail +.S um +Ġc ared +M or +Ġint uitive +Ġj ournals +_ IT +ĠT rou +ä¼ ł +Has ColumnName +Com posite +Ġsp ice +_d isk +_CODE S +ĠInt roduced +ion a +Ġnue stra +o ct +ĠĠĠĠĊĠĠĠĠĊ ĠĠĠĠĊ +(param eter +Ġstud ios +Ġproject Id +Ġbd sm +.Sql Client +im izer +ĠC ARD ++ t +a an +.s ol +_Ad just +Ġright eous +ĠLog ging +.f ilters +_T AB +ĉs ys +roph ic +other apy +ĠB rowse +key board +R ON ++ \ +ro pped +Ġext ensively +f k +Ġl ime +year s +Ex c +Ġs ph +Ġche ating +and ro +ÃŃ o +Ġpr ince +o ire +ĠD estination +ĠConvert s +Ġup stream +o led +Ġserv ants +Ġsem antic +Ġcr unch +Ġevent ual +run ner +/ error +Sp in +Ġsecret ly +Ġas semble +.P erson +end error +_ < +Ġp endant +S leep +ĠChem istry +Ġboss es +l k +)) ),Ċ +Block ly +DE VICE +Ġreflect ing +Ġam ple +Mill iseconds +ĠPresident ial +Ġus uarios +ĠN Z +ĠSal ary +ĠA manda +_n p +j ury +Ġkö n +Ġtherap ist +Ġhomosex ual +ĠDr ake +-w indow +ĠLoc ated +.D river +ĠV IDEO +Ġmerch ants +ĠC hest +- lock +/ php +Ġmil ano +_ST YLE +arg er +ide a +G UID +adv anced +me al +Options ItemSelected +=' % +ĠCh am +: data +(st at +Will Appear +Ġinform al +aj i +Ġre productive +ĠC AS +ãģ £ +F UNC +ĠR uth +)+ ( +CON ST +ĠF ans +Ġgroup Id +xffff ffff +Ġsam pler +Ġ}} "> +. the +Ġh ollow +W AY +ĠFac ulty +Attrib utedString +ĠLook s +ĠR ex +j k +ĠM IL +Ġb ard +.L ong +Ġliv est +Ġsk al +ic ism +MA IN +Ġmu cho +B ODY +Ġes e +ĉ use +F oot +.SQL Exception +Ġinherit ance +re ceived +Ġput as +ed is +als a +ĠError Message +Book ing +Ġtr act +ac z +ĠC ant +_reg ex +Ġide ological +Ġj ihad +h os +/s ys +col m +(p ool +Ġest án +ĠP ending +em ás +Ġktó ry +));ĊĊ Ċ +trans actions +Ġw ield +it ere +ert ure +_s s +Ġstretch ing +Ġprison er +.Read All +Ġbes ch +-- ;čĊ +Ġcr isp +_SC AN +Ġa e +Str ict +ĠMin neapolis +ĠBo eing +ar is +re k +_p ipe +Ġpri ests +(E IF +eh icles +ĠInter active +b etween +ĉNull Check +ĠBl air +ĠL t +_in line +eth yl + ¼ +_p ackages +Ġbarrel s +_ he +Ġreg exp +_ pts +_H andler +ing ular +ĠN issan +ĠR anch +Ġper ch +Un supported +Sm ith +ĠLeg ends +M i +Ġg f +st eder +Ġacqu iring +Ġsim ulator +() ," +re ceive +Ġin place +A CTION +ĠWeb Driver +files ystem +< Order +lo pen +ĠHE IGHT +.set Border +į ° +__ [" +Ġcl amp +Seg oe +b ands +to List +amb a +>' +Ċ +Ġcred ible +am at +play ing +.setImage Resource +qu el +Ġpod r +ge om +E k +ĠQ atar +Ġg eld +? ',Ċ +Ġc yl +( ax +ĠW I +ur ally +ĠBr asil +Ġsen za +ale y +on en +Ġb ah +Ġmolec ule +R ad +è¿ ° +AN CH +- background +- agent +Ġprol ifer +: boolean +Ġt ide +erial izer +_ ;čĊ +F ee +** ) +erg y +ĠHon or +.Log ging +ir is +Ġunder mine +ĠD y +Ġt yr +Ġde que +Ġdam er +([] )Ċ +.layout ControlItem +pe ated +C AN +rag ments +L and +) ]);Ċ +ĠS ah +ĠDE CL +With in +ĠN amespace +an other +sem bling +.des cribe +Con sum +ĠF ear +g iven +Or ange +< boolean +Ġstead ily +pa Repository +Ġresult Set +_ ENTER +_re peat +Ġt ones +ĠPRO P +n al +part icle +Ġsign aling +Ġaccess ory +ĉĉĉĉĉĉ ĠĠ +Ġvie le +ĠNo ah +- ag +Ġmur ders +Ġa ired +ĠPL AY +ĠS ullivan +_C ore +Ġul ong +Ġblog ging +> This +Ġdata Index +Ġprint able +ĠE yes +_target s +(P y +. over +Ġbr u +am pton +Ġplaint iff +< Key +b ull +Ġ⣠¨ +Iss ue +.cor nerRadius +C ritical +_p hi +. angle +Ġdynam ically +! ");čĊ +> );Ċ +in vest +.* ĊĊ +Ġt élé +Ġsuper f +Ġcas cade +DT D +Ġviv id +Ġsubsid ies +ĠH ass +Ġcoll aps +Ġcer amic +{} ". +ĠLeak age +-tr ash +coll apsed +-s ocial +ĠCh ad +Ġincl ined +Ġst o +Ġstory board +.p ayment +stack overflow +ĠRaid ers +Ġ# ' +olic ies +ìľ¼ ë¡ľ +em ap +Ġk j +Ġqu ota +ĠGard ens +ë² Ī +ĠAng els +Ġof t +Ġlower case +Ġi Param +Ġche apest +un ta +_p kt +ic ators +Ġle urs +Ġdecre ases +ĉ define +PRE C +amm ers +ĠPre paredStatement +(d irection +Ġcre ws +ark ed +ĠMem phis +ĠS ell +G TK +Ġm aid +: disable +éĽ Ĩ +ĠP f +Ġal beit +open h +?> ">Ċ +.get Source +(s cale +D u +ĠP IL +_ref resh +Ġbet s +(c ar +ĠV on +| --------------------------------------------------------------------------Ċ +ĠGr at +M uch +( Dialog +.stop Propagation +Ġte k +Ġex its +'], $ +Ġphone Number +uc s +ec imal +------------ -- +in p +.po jo +Ġcor pus +Ġpractition ers +.p ic +" testing +Ġstring By +.Not Null +Ġr ang +.D ynamic +_R ender +аÑĤ а +Wait ing +ĠW ik +Ġoverwhel med +% "> +ĠA E +}} >Ċ +u w +_t yp +Ġbuck ets +Ġgre eting +Ġla ughter +Ġant agon +uggest ion +- email +ĉt op +Ġer os +_tr i +Ġiss uing +Ġh á +Ġisol ate +Over flow +, E +Ġnut ritional +ĠAbb ott +Ġn f +.t ouch +.fetch all +_z ip +") }Ċ +Ġam at +ĠC isco +Ġn Ã¥ +PLE X +Ġse i +f oto +.to Json +å¤ ļ +ĠKle in +Ġlib c +Ġmin ers +å ¢ +- print +ĠP ride +T odos +Ġmask ed +Ġset Data +Ġtele fon +Ġunh appy +ĠT ables +ge b +( debug +_all owed +- access +Ġlog istics +Ġg ems +ĠM ature +Ġr sp +ĠAl le +.get Bytes +\ web +ynchron ized +Par agraph +Ġth rottle +.sql ite +cons ulta +ĠSe ah +C e +Ġsub mar +ER E +V ous +Ġre ddit +Ġsql alchemy +-m ile +oc ide +P our +}} ">Ċ +st ead +Ġ@ ( +Ġ[ ]) +ĠAd s +Ġover load +r idden +ĠDes ert +ĠW rap +ĠPortug uese +et z +ĉf irst +Ġmile stone +æĹ ł +Ñĥ Ñī +(s uccess +< Vector +co ol +Ġ[ ]);Ċ +erv als +Ġin vert +" io +cur so +fr agment +Ġfeas ible +.set Position +Ġel m +Ġimag in +@ Spring +Ġb ats +pu és +ga lement +ns ic +gi ene +ell ation +ĠBa iley +Sh ar +ĠT ul +ĠH K +Ġfree zing +gl m +ce ans +-c ut +_c ircle +åij ĺ +n egative +Ġind ian +s alt +Ġt ing +ĉm od +Ġs int +ak in +um l +ĠText Input +Ġpop ped +T MP +Ġpark ed +×Ļ × +ĠF usion +Ġhe ater +ET F +ro zen +h all +ĠM ik +lev ard +- heart +ĉ order +M aking +Ġpled ged +Ġdir s +$ post +ĠH err +stant iate +, "Ċ +.get Color +ĠS AT +Ġtimed elta +ĠM ai +ĉm ethod +Ġid iot +ĠTr av +ident ified +ĠDiv ine +.get Path +D ash +Ġinf iltr +Ġhandle Submit +bro ok +.g eneric +.short cuts +................................ ................................ +Ġdat ings +ĠM V + # +} "ĊĊ +Ġimprison ment +ason ic +rou d +uc ion +æĬ ¥ +Ġdia lect +Ġon Mouse +const expr +.label Control +Ġwe aker +Ġman kind +ĠRE CE +Ġd iz +Ġapp Bar +Ġqu é +f ra +_default s +Ġal iqu +_at om +: indexPath +Ġmiss es +Ġvis ually +ĠH ands +STR U +i ates +_ asset +F inder +mid t +Ġsn acks +(__ (' +. uri +ĠIn strument +ven ir +($ __ +.Dot NetBar +Ġconfig s +Ġguess ed +ि ठ+Ġinitial izer +Ġ? ", +ĠVer izon +man ifest +ge ben +.d etails +G ate +pons ible +ĠEl im +, str +Ġwrit ings +ĠD erek +ĠCo ordinator +Ġpill ow +Ġnotice able +R s +Ġduplic ates +ern els +k J +.z z +oll and +ĠSE CTION +_f name +uff led +'].' ")Ċ +ĠD ollar +Ġem oji +Car ousel +- player +Ġadjust ing +Ġjug a +alleng es +g ene +(body Parser +lop edia +ĠBeh ind +Ġslee ves +Ġdrag ging +ĠChe vrolet +Ġb iz +iv ities +ĠFrequ ency +, char +.W HITE +_pre view +) ';Ċ +_ ax +ION S +.c pu +.input s +UB E +_fe ed +ĠSup plement +! ). +es us +ĠU DP +Ġmicro phone +Ġconf irms +.is NotEmpty +":" ",Ċ +_S CREEN +ĉ expected ++-+- +-+- +ĠH ait +fast call +Ġdep ict +v b +_p icture +ĉd escription +ĠW ife +uc i +Ġv icious +ä» ĸ +ue ba +Ġset User +ãģ ¡ +Ġd iving +Ġoper a +user content +ar ah +) }, +y un +vel t +Ġun covered +Ġh ips +Ġosc ill +Ġassert ing +ĠX i +.re store +ke a +Ġsp elling +Ġder ive +ab we +ĠD ow +.set Type +_v s +Ġco zy +.c ategories +O rg +_m gr +Ġd ungeon +collection View +ĠBl ank +ac ias +ä ä +_clean up +_ACT IVITY +Ġtri angles +.Menu Item +Ġip hone +ĠW on +] ]ĊĊ +ĠCompar ison +.D oc +Ġcan onical +ĠSud an +') { +Up Inside +b uiltin +ENC Y +x be +Ġch uck +Ġcontrad ict +Ġnuest ro +Ġarchitect ural +ĠF ib +Ġcomp ares +* k +C fg +çĦ ¡ +nt en +Match es +ĠDOWN LOAD +_HAND LER +man agement +[ S +EN G +ÂĢ  +f ang +Ġsl ipped +ĠL anka +esc aping +Ġtack les +ĠPed ro +.P rop +.' ' +.G enerated +.New Guid +at rigesimal +ill on +Ġstat istic +spec ies +hold ing +Dr upal +Ġfundament ally +Ġbond age +Ġres olutions +Inline Data +\ Type +est ion +.w rap +Ġwar riors +ĠLOC AL +Arch ive +Ġembr aced +á» § +.V er +ĠAff ordable +oles ale +ĠAp plied +ĠCon version +m ega +_c am +Ġcer emon +aur us +ĠVol k +.op ens +/ about +ĠSt d +j ournal +()) {čĊ +," \ +( Arrays +ĠD ense +ase ña +än ner +/ stat +user Data +Ġg erman +Ġt z +worth y +Format Exception +ph erd +Ġsm iles +ĠWh enever +( adapter +.bad logic +Ġbrief ing +.Grid Column +- char +dim ension +ĠC opper +Ġnin th +Ġ' {{ +Ġr av +_T able +Ġderiv atives +ĠR aise +ĠF ut +arm or +-p adding +Ġre min +ĉ style +ĠMembers hip +Ġspread s +Ġgall eries +ĠClar ke +Ġcon ception +min ute +Ġab usive +_ad j +Ġterr ific +Ġover t +our cing +Ġentr ada +level s +Ġcrit ique +Ġrespect s +ĠM MA +i ene +Ġenc aps +ĠRay mond +Div ider +iv able +b az +Ġ@ _;Ċ +ĠCl aire +Ġur ging +CE E +Ġtransform er +disc ord +ĠJ ourney +t os +Ġcompet itions +ĠO BJ +ĠB is +Ġrelax ation +id y +_IN STANCE +ĠP ref +d ados +ici encies +ĠMedia Query +ĠC ube +ĠStr ange +g pu +(d ays +_Init Struct +Ġfinger print +em at +ĠGe cko +Ġr ails +ĠL um +str action +ig ung +(m ovie +_d ictionary +_int errupt +ĠQ C +ik ed +append Child +rec ipient +r é +V e +Ġtow el +.last IndexOf +Ġplace bo +ĠW ie +.es p +( Debug +oper ative +Ġdece ased +& id +ĉm utex +el ic +Ġb apt +ĉ čĊčĊ +Ġfar ther +H alf +.dis able +.menu Strip +le ccion +Ġresult Code +Ġc ans +-e lection +f emale +_F IX +aus ible +ĠP OWER +Ġrecon struction +Ġsc ans +.Xtra Bars +âĢĺ s +Rem oved +Ġparagraph s +_m argin +Ġl ymph +Ġb os +ling ton +ĠBapt ist +Ġadvertis ements +ĠMan age +/ yyyy +IO US +ENC ES +ĠF iction +ĉm enu +ĠFile OutputStream +ov an +ĠF eng +Ġsk ipping +get Class +ann i +Ġreb ounds +Ġpublic ity +Ġing res +use ment +Ġthought ful +.Ch art +Ġhat te +pass port +Ġhook ed +ĠL ens +Ġflag ship +Ġst ip +ĠG EN +Ġcl ues +ip v +ĠR ise +ĠG ew +tab lename +Ġfore most +_ validate +_an alysis +oll a +Ġqual ifications +Ġdistrib utions +ĠFl ower +Ġt ense +Ġthank ful +Ġcl utch +Ġun ified +ro ads +Ġsit i +Ġst all +_P RIORITY +c stdlib +_USER NAME +.by tes +? page +ermal ink +ĠVe get +/v nd +- author +.N ONE +ĠCon current +ĠC ry +Ġstart ers +ĠInter action +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +ĠLE VEL +E ll +Ġcom boBox +ĠTh eresa +te k +_H andle +Ġab y +.g dx +, end +(L ocal +O l +kn ife +ar ial +ĠH off +Ġprostituer ade +Do ctor +Inst ances +.Set Value +ĉf rom +Ġlux urious +Ind ent +Alloc ator +_D RAW +(", ", +ĠFr ances +Ġgroup Box +(s chema +Print f +OR IES +- gradient +Ġre put +ar in +_D ONE +in cre +ig nty +Ġex ert +Ġ- . +/ App +-th rough +Ġdecl ining +Ġdess ert +Ġinc umb +Ġdesign ation +.P ORT +, strong +Ġsand box +Ġw ines +ĠP av +$ str +ask ell +Ġh ö +ĠP Y +Get Instance +Text Input +game Object +/ events +created At +Ġlocal Var +ĠWH ITE +per ed +ile ge +eff icient +, color +c ate +ĠC afe +Ġsimilar ities +Ġp umps +ĠHung ary +.User name +Ġsk ate +Ġtouchdown s +Ġacceler ate +ĠH elen +OM EM +ĠK un +_v ol +Ġfind All +ĠMens chen +a head +); " +kom men +Ġpossess ed +.arg max +.trans ition +AR P +OLUM E +(s cript +ĠÐ ĺ +ĠF inding +on ces +I o +B old +Ġrenew al +_D IALOG +Ġdis reg +INT ERN +Ġt oute +Ġelect r +ĠG ross +ĉ true +.F ields +ĠW IDTH +ĠD ent +Ġà ģ +NS Notification +Ġa os +Ġme lee +. Validation +ĠDE C +-depend ent +Ġsu ic +T raits +$ message +ĠD ear +ĉ FILE +l anguages +.P rot +.add r +-g eneration +IC ON +Ġtrans plant +-d escription +Ġch asing +Ġche es +Ġ} */Ċ +Tr ad +qu eries +/widget s +sub package +Ġes pec +Ġcr acked +Ġcompet itor +P urchase +- team +olec ular +or Thunk +& P +Ġrel ent +/ #{ +Ġproduct Id +Ġè ¾ +ĠL av +ĠAl ter +.M ode +AD IO +gr p +æ ·»åĬł +Qu it +Ġdepth s +-c ategory +ĠD ATABASE +S PELL +ĠFal con +ĠQString List +Ġ'' . +ĠIn stitution +d amage +az or +bel ongsTo +ver ages +ĠN ONE +ipp ets +, \Ċ +Ġfoot print +_ archive +n ak +.get Field +ĠRef lection +Ġ' ] +ĠH BO +_dis count +Ġin cest +ĠD odge +ĠW ade +.N O +" encoding +ĠBlock chain +Ġlaws uits +ĠM aint +ch ten +Ġét ait +Ġktó re +_ ctl +(t imer +B attle +iz o +ay ed +I OR +ĠGlas gow +Ġsyn th +_log s +.p ose +_Adjust orThunk +(( & +Ġuns ure +yst ate +íķĺ ëĬĶ +O ULD +. ng +Ġdefault dict +work space +Ġselect ive +Picker Controller +YNAM IC +.method s +Ġpath ways +ĠF ew +K G +CRY PT +follow ing +ĠD LC +ĠS ara +Ġpres et +estruct or +ĠK urt +Ġair plane +Ġo mp +ĠParent s +ĠMart inez +.com plete +Ġbroad ly +Ġsc are +ĠM é +Ġelim ination +Ġpou red +/ sw +Ġcom un +Ġm asc +ĠOrgan ic +ĠString Utils +il ateral +Ġreluct ant +- age +Ġn z +." \ +Ġpast or +ale z +Ġe fect +pro v +/ init +Ġp enn +und s +Ġs size +ĠPro j +bas ename +Ġsh ells +ĠNe ck +ĠEn forcement +vid ed +st own +S phere +$ r +uss en +af il +ĠTele gram +Ġanaly tical +нÑĭ е +us ually +x n +Ġhistor ian +ĠGreg ory +ol ph +ĠUn a +Ġcon tributes +% - +anti ago +ÑĢ ед +.reg ion +Ġab rupt +ĠUnsupported OperationException +ĠT ASK +_f inish +Ġnot orious +ĠV s +ĠM Q +Ġsun set +Ġun acceptable +ar cer +Ġill umin +ĠOr b +Ġb h +E ste +_dis patch +Ġr ipped +Ġtou jours +ĠPar cel +_ ll +.user Name +.class es +S OURCE +( Number +ел Ñı +Ġhead phones +(s ide +const itution +ann ah +čĊ ĠĠĠĠĠĠĠĠčĊ +Ġcl iff +- ref +Ġmo strar +ĠPow ell ++ y +ĠB G +_f ragment +.P ort +Ġreal izing +param ref +Ġh ometown +@ Table ++" --}}Ċ +F rench +Entity Manager +ĠPl ain +//////////////////////////////////////////////////////////////// //// + ³ +( RE +c apt +Ġorgan isms +Ġj ets +ol ocation +ĠApp RoutingModule +Ġgl orious +æľ į +Ġdisc arded +ĉĉĉĉ ĠĠĠĠĠ +ĠArn old +l ug +Ġpar l +Ġhorm ones +Ġm ah +ĠSon ic +Ġorgan izers +_PL ATFORM +.in v +Ġch ord +vent ional +ĉ of +Ep isode +. Enum +unk t +ĠD h +ĠJ ared +ĠN ak +Ġint ends +End ian +Ġa ustralia +_c v +(res olve +Ġclin ics +lik ed +ASH INGTON +in ha +' * +ĠN P +_b eh +Ġh f +Ġw ür +c ategoria +$ form +Ġsub way +Ġis Active +pop ular +C our +Ġco oldown +Ġa insi +ĠGL uint +ere al +Ġarray Of +Ġh atch +======== == +ress es +_P P +. ^ +_dec ay +ĠB less +met rics +ĠCOPY ING +ĠDump ster +ĠJos é +ĠDesign s +< +Ġ" }Ċ +time zone +Ġe er +max cdn +ĠE SC +ig aret +_conn ected +_re verse +Ġquestion able +ĠUS C +Ġtut ti +Ġdrop out +ĠActiv ities +ĠW inds +')) );Ċ +Ġcon gest +ÄŁ ı +Ġprolong ed +è¿ Ļ +ĠCross AxisAlignment +LE EP +ĠVAL ID +ĠG az +Ġdepend ence +ĠP rix +.Compiler Services +j ump +Ġstr at +c irc +ĠC USTOM +x aa +Ġb mp +Ġb ureau +Ġw aren +N X +( Window +ĠChrist ie +_F E +Ġt n +ĠOm ega +communic ations +Home Page +com pletion +Ġsupply ing +YP ES +á vel +åĪ ¶ +(c lick +\ Contracts +/ questions +Ġe z +AM S +.m esh +Ġ' \Ċ +Rob ot +Json Object +ĠD F +ĠProcess or +_sh ould +.prot obuf +- users +Ġemb ry +F ONT +Ġstart ups +ĠData Source +) # +uro s +_C olor +Ġstand alone +} [ +j d +Ġforg ive +Ġng x +ĠGener ally +Ġconfig urable +/ order +Ġv as +') ";Ċ +ĠR R +ĠT roy +Ġcomprom ised +ĠSw an +int endent +Cent ral +_ keeper +Ġar quivo +ĠRead Only +_cur ve +k v +ent in +è ± +ĠE y +.im read +ĠP am +if fe +at ivity +xb c +Ġgr im +-f illed +names e +'] : +Ġa ur +ĠGib son +.Mouse Event +Ġl ado +avad oc +Ġfam il +ĠM oder +f ps +ãĢĢ ãĢĢ +- example +ĠAl zheimer +ĠU tf +_arg uments +Con clusion +text Content +rem aining +Ġinterrupt s +ĠBack up +ĠM ong +Ġrecept ors +h istor +.cor outines +Ġsh outed +Al arm +Ġcomb ust +Ġg rote +ult ural +( ids +---------------------------------------------------------------- ---------------- +ipl inary +O pts +ĠY ale +local Storage +Ġequ ival +ĠF leet +\ b +* pi +ĠQ Label +æ ¡ +Ġv x +ĠA CL +Ġsu cesso +Ġper c +ĠNot re +Ġan arch +R ing +sp b +Ġstr pos +st ores +ĠMap le +(Main Activity +(" ")) +Ġview Holder +Qu ad +Ġig ual +ors che +.m argin +Ġind ie +Ġfr anc +ĠForm Builder +ĠPart icip +.fl ash +Ġstorm s +U lt +Ġf en +[ new +E ver +=" Ċ +Ġlocal ized +_f ollow +Ġn ave +Ġdomin ance +(t ile +J ournal +ĠV C +Ġpenet ration +ï¼ ķ +Ġcomp artment +Ġb ids +Form atted +****** /ĊĊ +(c ity +âĢĶ it +[ C +Ġuse Callback +a ub +) ?. +ĠV AR +ĠSe bastian +ĠM oss +Ġabund ant +G reg +ÑĤ а +_c i +Ġbib li +CR M +ĠAt tempt +ism e +d ash +ãĢ İ +_m u +.Formatting Enabled +Ind eed +-d irect +Ġsuck ing +Ġp ne +ocab ulary +ĠPack ers +.N avigation +Ġp ied +cri bing +ĠSt uart +.To Double +ĠSecond ary +S aving +ĠD ut +ĠM add +M agic +, H +.document Element +ĠB ST +Ġdiff ers +Ġmore over +_ nd +SE ARCH +п ÑĢав +æ ´ +to Match +Ġdecre asing +-m ember +amp us +( boost +D aily +Data GridView +ĠHttp Context +Ġh ipp +_work ers +-l anguage +é ĵ +Ġconsist ed +ath ing +ĠMer cury +$ content +Ġpract iced +ĠMod ules +_D AY +Ġweakness es +ĠL odge +Ġn ar +ĠM ate +Ġj p +ĠHttp Headers +Ġsm o +ĠT OKEN +] )( +Ġaqu i +sw agen +Ġs rv +ĉ ans +A round +ĠMan uel +Ġfiction al +ĠIM G +Ġ. ' +ĠB erry +Ġwall paper +sex ual +ier o +Ġ çļĦ +ìĨ Į +Backing Field +ĠAd rian +BASE PATH +Ġrepe ats +Ġbl ues +Ġunp redict +_c oll +st acle +ĠT umblr +ĠEl f +Ġass urance +Ġc ensus +ĠIM PORT +END ER +an os +Ġ= ( +ĠEll is +" ĊĊĊĊ +.w in +ĠA bove +al on +_t ick +Ġrepresent ations +Ġæ ķ +w id +ĠAr ms +List a +_f ailure +_c m +.Flat Appearance +Ġthr one +P atch +ĠV oy +eng l +Ġnegot iating +> ` +Ġshoot s +ĠF PS +.Y ear +ĠK iss +enc ión +reet ing +From File +Ġresign ation +Ø · +Ġtw ins +Æ°á» £ +Ġge bru +.get Content +.T ree +ĠEmploy ees +ĠF IFA +Ġcert ainty +(C l +Ġtot als +edit able +ॠĢ +.Report ing +M as +qu iet +.r ules +ĠV O +con exion +, K +Ġalloc ator +ĠPow der +\ Repository +Be at +_t ipo +Ġ[' ', +_IN TR +Ġ<< < +< hr +") == +ugg age +ĠC raw +Ġé galement +Ġg inger +Ġprim era +Ġprod uto +lt k +.User Name +Ġstr error +m ith +_n b +Ġdis comfort +']; ?> ");čĊ +drop IfExists +ĠB eg +_H AL +Ġcross AxisAlignment +ĠE vidence +Ġpec uliar +Ġinstit ute +ve is +Ġf ft +à ģ +Ġzo ekt +an aly +ĠHom eland +Ġpen etr +udden ly +ĉ element +ĠB ren +ĠTr udeau +ĠCub an +j am +us lim +_e v +Ġst ems +} % +Ŀ å§ĭ +Ġbrand ing +Ġcorrespond ence +.j query +¢ åįķ +ĠRead s +(Http StatusCode +ass in +(s lot +ĠGrad uate +/// < +Ġinform ations +EN ABLE +Ġp uis +Ġfind er +ĠBr is +Ġnett steder +_m id +Ġo gs +ĠSter ling +Ġar rog +str ftime +| ĊĊ +Ġvo x +ĠReg ardless +Ġes o +ĠCom fort +.Boolean Field +Ġu h +AC Y +Ġsque ez +ĠV ic +cont ro +. lo +Ġ ire +ĠCom edy +ë ¶ +Ġorigin ated +Ġsh ipment +| max +_g uid +lev ation +на Ñı +( undefined +ĠD DR +Ġshoot ings +ĠLat ino +END OR +Ġaver aging +Ġgre eted +Ġthe aters +о е +Ġd B +Ġg st +Ġdef inite +. Storage +.h er +Ġa fore +ĠRe ality +ĠGod s +vers ed +Ġhands ome +Ġex cluding +( ad +Qu otes +ĠS cheme +? q +ĠT amil +T icks +Ġp est +' n +Ġporn ography +_mod al +Ġ ---------- +Ġdis posable +F REE +Ġsh ark +C HE +Ġdep icted +Ġdemonstr ations +ĠK illed +ĠR ULE +Ġobs essed +Ġsimpl ified +Post al +Ġconcept ual +Ġp st +L as +_PRO JECT +ucceed ed +ol u +ÄŁ i +Ġpersonal ities +Ġres hape +Ġenc losed +ĉp tr +Ġtutor ials +Ġexpl oded +_DIRECT ORY +åĨħ 容 +Ġcan on +Ġrecogn ise +P AD +ĠAppro x +ĠRest ore +ĠImport ant +Ġheav ier +.Se quential +Ear th +ĠMil k +.set Request +.t em +Ġre construct +Ġskept ical +_Pr ivate +BU F +qu a +: a +Ġse k +Ġd well +oss a +Ġreward ed +и й +(top ic +_part ition +Ġ__ ________________ +Key words +ĠFr anco +L ite +Ġn aken +Ġз а +O BJECT +Ġcraft s +ĠSw ap +.X na +.Con nect +Ġbalcon y +(re al +ĠBarn es +b ir +ĠTw enty +ay an +at ars +ĠProp el +ĠIh nen +Up grade +Ġcur b +- second +Ġn eph +.p res +ìŀ ħ +.se q +Ġp added +" ? +j l +ãĥ ¬ +') a +Co ordinates +Ġen acted +ENT S +Ġl ac +.f inal +ĠPhp Storm +c alled +Ġin quiries +.m iddleware +ĠD owntown +/ ';Ċ +Ġkil omet +ac cel +Ġqu ien +w string +set Data +Ġman era +Ġmod ular +rim p +Ġtar iffs +âĢĻ il +_TH ROW +/c olor +ĠHT MLElement +Ġcar ro +Ġpr ere +Ġplot ting +ĠPos itive +ĠMach ines +OT ES +á» Ľ +ple asant +Ġal te +Ġa inda +th ese +Ġc ors +ip ay +ĠAdvis ory +ĠRub io +j q +Ġl imestone +Ġdet ached +设 ç½® +ten ant +ĠDep th +al ore +ĠÑģÑĤÑĢ ок +ĠF ORE +ĠL ay +p resentation +) ');Ċ +.sub plots +Ï ĥ +N OW +G ar +hand les +ab ra +put ies +ĠElect rical +M iddle +rop ic +ĠJ D +ĠD yn +ĠB ristol +ĠMc Carthy +Ġstri ker +Ġenumer able +ĠEv an +.default s +qu ences +) || +ĉt oken +â Ĺı +-d ropdown +ST ORE +ĠGraph ic +( pp +Ex pl +Ġup wards +ĠD istributed +ĠW EB +J er +is NaN +çĶŁ æĪIJ +> R +üss en +ef s +Ġun cover +Ġl ud +.cal culate +Ġint ptr +Ġmidfield er +. Headers +Ġm f +ere f +.M etro +ĠSpe aking +: b +Ġcryptoc urrencies +Ġdem ons +ĉ EXPECT +Ġw icked +y outube +: Int +ĠHind i +ĠC AT +ĠØ ¹ +r ar +om ore +/ per +/lic ense +Ġre im +Ġawait ing +Ġle thal +ĠE F +round ed +ĠPl atinum +ĠвÑģ е +.co ords +.De vice +/ item +ĠW enn +compile Components +ĠK inder +.remove Item +Ġand a +bn b +Ġpr a +( transaction +Ġembarrass ing +ĉ BOOL +.content View +Ġevent data +at ore +Ġprovided In +ir ma +Ġz ona +_H W +æ Ļ +Ġst ove +Ġcounter part +_Pro duct +_MAN AGER +Ġinfr ing +ĠE RA +_p arty +Ñ ij +Ġin ici +_ Request +Ġmir acle +Ġcancel Button +S py +at ó +Ġpol ish +ĠNic ole +.display Name +\Request s +Ġuse History +Router Module +Ġst ared +ID ER +Ñĥнк ÑĨи +Ġnot a +$ arr +pec ified +Ġto pp +_DR IVER +/ ng +å ł +_t m +% timeout +< s +Ġ( *) +ĠHttp Request +_TR ACK +(n ote +ĠExp lore +_s erv +Ġç » +B inder ++ ", +. att +ĠEth i +Ġc ódigo +=' \ +.l ines +( Of +å° Ĩ +miss ible +Ġv é +Ġac oustic +Ġcraft ing +n it +.b a +ĠLuc y +Ġi Pod +Ġpup ils +-m ax +_w r +(c p +ĠRE PORT +Ġd ns +ĠRe ferences +Ġundert aken +Ġkø benhavn +Ġch ai +ĠC roat +_ Log +rown ed +_m ed +ĉ date +# __ +Ġcost umes +ĠRe quires +aff le +ç Ĭ¶æĢģ +-S emit +ela ide +еÑĤ од +Ġp estic +Ġd ra +DOC UMENT +Ġ... čĊ +}` }Ċ +ĠA uction +ĠD ock +xxxx xxxx +(get String +ħ į +Ġborder Width +ĠMach inery +Ġpredict able +.S H +Ġam plitude +.for Root +IN avigation +Table Model +at trib +Ġmaneu ver +Ġexc av +B ERS +Ġd apat +Ġinstall ations +.A sync +Ġr ays += âĢĿ +; ččĊ +.c rypto +_db g +ĠEnum erable +Of Size +_epoch s +m w +M ENU +out line +ĠP apers +============ Ċ +Ġuniform s +ĠG ig +- package +ĠJen kins +ĠHome Page +.is Selected +Ġmechan ic +M K +ĠS ounds +//---------------------------------------------------------------------------- -Ċ +Ġresearch ing +Ġinf os +ograph ics +ers et +([' / +ĠTim ber +. agent +.to JSON +_command s +par ing +_ad just +.n ome +(g lm +Status Bar +file path +? âĢĻ +Ġdetect ive +Ġunser er +ĠTib et +EN DED +(se ed +Ġsne ak +Ġam or +=" // +ĠPan thers +all ax +ĠL IVE +ĉD WORD +]= - +Ġtorn ado +/ min +Ġlung s +-c urrent +ĠBook ing +åĪĹ è¡¨ +Ġenjoy ment +ठ° +J A +typ ed +.B tn +f at +ug al +ĠSh ares +Ġdis gr +ĠB AR +ĠFO X +Op code +ĠS z +key down +iction aries +Ġdetail ing +} ))Ċ +Ġp ok +Ġdemonstr ating +Ġnot ation +l ayers +@ if +ĠN PR +.strict Equal +ĠRec ipes +.T ensor +Ġliqu or +Ġdeb ts +.ends With +W heel +.P os +CS V +$ arity +Ġun stable +( loss +ENS OR +Ġele ven +ĠL opez +ĠHop kins +con om +ĠS eth +Ġpo ems +Qu ant +Ġg sl +Ġsy rup +Ġs ibling +Ġc ass +-v ous +ö t +_P ATTERN +_SE CTION +est imated +up grade +.m ongodb +ĠBo at +_C TX +Ġfetch ing +ust in +pi el +M arg +Ref lection +Ġd uct +ĠMunicip al +Ġb x +.Get Current +ml ink +ĠAccount ing +ĠGene va +_P os +Ġpass er +Ġhear ings +com pan +Ġfrag ile +Initial izer +walk er +.M aterial +ĠHun ting +trys ide +Ġk at +Ġcl erk +á Ł +do ing +ĉg roup +Ġsan ction +.l b +ĠL azy +ĠCon straint +P agination +Ġpou vez +ĠInd icates +M ER +Ġcour s +Ġyear ly +Ġgros se +abb rev +ĠD ON +Ġproceed ed +ent lich +Ġproperty Name +ĠTe aching +st adt +Ġc utoff +orn ers +Ġa frica +Ġrend ers +ĠYan kees +ĠTool bar +sp aces +.fill Style +Ġseg undo +_str len +.F irebase +å¤ Ħ +Ġmention ing +\ ( +ĠVal ve +Set ter +Ġsp ans +ĠAl cohol +ĠLet ters +\x e +ĠT K +_B LE +.get Result +< Player +ĠP att +Ġeas ing +Ġtur key +ĠF en +') " +Ġconf ined +Ġin clus +Sup erview +(with Identifier +enc ial +Ġstuff ed +Th eta +Ġeconom ists +} ));ĊĊ +co okies +ĠRo ose +ĠChe ese +Ġfich ier +Ġen forced +AB B +no ÅĽci +_AL LOW +Ġrecru ited +Ġexpend iture +-n ight +Ġassert NotNull +_ex ecute +ĠØ ¯ +IN DEX +_F MT +Ġresc ued +ĠMonth ly +ĠCons ervation +ĠG eb +Ob ama +Ep och +ic ies +ĠOr t +Ġso it +( icon +F riends +m ol +Ġground ed +ĠC ause +ad ena +WE EN +ĠL un +IT IVE +. loop +_un til +Ġcor r +.ed ges +Ġhyp oth +ched uling +trans lator +ĠÐ ľ +R om +ãĢij ĊĊ +ĠX amarin +Ġviol ating +. anchor +--- ĊĊ +Ġtr ader +AD VERTISEMENT +Ġuns ere +ĠD AO +Ġbl ond +ĠP AT +.g lob +Ġè¾ ĵ +Ġsplit ting +Ġun subscribe +Ġatmos pheric +ĠTr im +Ġcit ation +Ġin ference +ĠF t +ĠDar win +find One +ĠG el +( Convert +Ġaccess or +; text +(s orted +Ġjud ged +); \ +: p +Ġme ine +ĠS lim +.Command s +Ġper ceive +coh olic +< Data +.entry Set +Ġassert False +ĠPat rol +ense m +ÅĤ Äħ +¨ ¡ +W IDTH +ĠRes cue +ĠU IF +_THRESH OLD +ĠMich el +ATER IAL +opens ource +ĠD iana +Ġinv ites +_B ODY +Ġreserv oir +Ġro i +c ust +(t c +ï¼ģ ");Ċ +Ġfest ivals +Ġperform ers +Ġclim bed +Ġj ungle +String Length +Ġunlaw ful +ier re +vertis ement +Ġst akes +Ġh ats +Mod ify +ĠLET TER +.H ide +Ġstat utory +_ white +ĠPer l +uten berg +em ple +.W orld +Ġoverlook ed +Ġcon cludes +/* ================================================================ +-w ise +ĉ stream +pop ulation +Ġevent o +Ġillustr ations +ft s +Ġaut of +ĠPro cedure +Ġdes erved +-t imes +Ġg ol +N SError +cre st +ĠPak istani +any ch +get Current +Ġl ar +nt l +ĠRe becca +Ġm ateria +Ġfind By +/ ad +Callback s +ĠAl s +ĠKat ie +ĠObservable Collection +ĠDocument ation +Typ ed +ĠCulture Info +ĠTim othy +Ġlater al +" type +Ġun authorized +Ġteach ings +Ġdebug ger +[ value +Ġal ors +Ġu z +Ġsc atter +Ġdown ward +Ġmig li +status Code +Ġ( )) +ĠM W +Ġм ож +RO SS +.b uf +Ġfair y +ĠInf rastructure +=> " +t lement +$ (" +From String +ĠB ild +Ġconvent ions +_n ative +ĠIns pector +ĠP ist +ub ar +Ġreg s +ĠP ilot +Th us +>' + +Ġc ela +.new s +( Product +L iving +R ussia +Ġfac et +et ical +Ġ[' $ +/ [ +ĠD ire +Ġg ases +ĠIN FORMATION +ĠE at +ĠFor ums +ĠChar acters +_m et +Ġìĭ ľ +Ġk ings +ach ie +ĠL ambda +Ġtim ers +ĠLight ing +ĠCase y +add ir +and ex +. answer +ĠH ip +ĠPr incip +Start Date +Ġ ãĢĮ +t res +Ġ& # +.Max Value +ĠPro blems +Ġlat ex +Of Class +ĠLyn n +// ' +Ġvoy age +Ġshut tle +ĠRoll er +ĠRuntime Error +uy a +D ic +ĉb uilder +Ġbul lying +Ġsimple st +.c alled +ĠL R +Ġmor ality +Ġst urdy +tr acking +.sw agger +_B IND +IT OR +-url encoded +ĠÑ ħ +ĠTr inity +Ġtr aps +Ġ| - +Ġset Text +Ġbarg ain +Ġbr akes +.get Code +Ġmigr ate +Ġrib bon +) return +Ġcharg er +ac om +ADI US +ĠAmb assador +-a fter +Ġann i +ĉs pin +Con cept +ĠHend erson +ĠH OST +.r ank +ĠNor theast +Ġber lin +Ġrequ is +.f eed +Ġsource Mapping +ĠRen contre +. ajax +nest js +Ġtre k +ĠN acional +Ġ& [ +Ġpay able +ort ex +Ġde pt +field Name +Ġcomple tes +ĠR VA +Ġon ions +al ignment +Form ats +Ġ' {$ +Hash Set +ĠB od +.Invariant Culture +Ġsettlement s +Ġhy dr +. updated +vent h +( seconds +="/ " +Ġweb page +( ĊĊ +Ġt ir +Ġto es +ĠBr ick +Ġamb ition +P ot += max +ET IME +Ġdep ot +c alls +ĠNor wegian +` : +Ġbur ger +Ġprofess ors +ĠAl locate +-third s +-ch art +Ġfor d +* N +.k otlin +Ġpaper work +ĠDE VICE +% @", +res pect +(m p +é «ĺ +- if +Ġcush ion +ob ot +Ġpar c +SP ACE +ĠNet anyahu +Ġself ish +fe at +Ġclient es +-to ols +Ġpor ch +Ġj q +. verbose +Ġlib erals +] )ĊĊĊ +p ies +Not Blank +( term +ÈĽ i +_Param s +.normal ize +B ullet +AS IC +(h ex +_client e ++ , +_D I +Ġforth coming +} ")]Ċ +se o +U m +> Name +Ġcomfort ably +irection al +W ITH +/ pr +ĠP oor +ĠVit amin +v ic +G H +Ġprior it +ĠN N +ĠC losed +¤ í +Ġis Open +\ Console +And Feel +.S UCCESS +_OPER ATION +pol ation +ĠT as +ps z +> '. +C URRENT +V endor +host s +ĠE rd +>tag ger +ĠsourceMapping URL +Ġmar athon +_c losed +Ġexem ption +Ġrecogn izes +ides how +' $ +('/ ');Ċ +m its +war z +ĠCh erry +µ ¬ +n or +port e +Ġw l +_back up +.get Boolean +.get Resource +Ġdefinit ive +. EditText +Ġs ÃŃ +.C ONT +ĠPL AYER +.c ards +ĠSh ore +('/ ')Ċ +cl uir +Web Driver +(m onth +-re lease +Ġins pector +å £ +ĠN F +_cl ip +åŃ IJ +Ġinteract ing +.t mp +Ġ'' 'ĊĊ +Ġde e +Ġfro st +"] ))Ċ +ĠPl aces +Th rows +f ork +/ day +i Phone +ĠM IC +Ġfold ing +Ġcro re +ĠCh iefs +pher ical +( price +.Write String +Ġexit ing +] ',Ċ +ight ing +Ing redient +( vertex +Ġscroll View +h f +: new +SE N +se ctor +Ġsp ins +ĠS cheduler +ote chn +sem icolon +Font OfSize +ĠSpecific ally +fl amm +.Object Id +Ġcont a +_per missions +ĉF ROM +IC ODE +/ kg +ĠHot els +-m ed +ĠD in +Ġn avy +get Param +Ġm end +Ġportray ed +ĠMet ropolitan +Paint er +Ġref erral +_g ood +Ġmar vel +osa ic +> (& +. ur +Ġest os +Will iam +Ġtim ber +Ġquel ques +ĠDoc uments +.X aml +Ġbatch es +éģ ĵ +ĠRe leased +T ail +CO OKIE +he id +_st ation +ĠV ia +S ale +ĠRe peat +Ġprom in +ĠZ o +- forward +ĠI on +it ary +Ġj us +- request +Ġproud ly +ĠStream ing +(Mouse Event +ĠS print +_ rotation +Re positories +Ġt art +ĠÑģ в +Ġm appings +è ª +C u +C ycle +Ġb un +ĉl ua +ãĥ ī +Ġ(( ! +Ġcollect ively +ĠCon d +Ġwsz yst +(l ib +openh agen +_s kip +.Column Header +é Ĥ +peri enced +ı è¿° +_p rops +Ġcontr ace +Ġmatch up +ab etic +.m embers +RE CT +(d at +Ġs og +ren om +_M ethod +Custom ers +full name +Z N +re try +Ġk ap +ĠNe u +è Ĭ +add Child +will Return +_p ermalink +Ġener getic +ĠW et +ĠMor r +Ġg cd +count s +, type +d ig +( Login +Ġcr acks +Ġbacter ial +ĠMe at +ĠArm strong +ĠBron ze +Ġapprox imate +_dir s +lig a +ÅĤ ad +Ġkind ness +Ġcont re +ĠE VERY +M ET +Ġannounc ements +g pio +ĠWaitFor Seconds +ĠPhotos hop +Ġdis contin +/ dd +Ġtop ology +an ical +. interface +auc oup +.Hash Set +ARI ANT +(r outes +ĠT eh +Ġh ype +] "). +Ġsl am +Ġbro th +- inter +ĠR id +-m anager +Cancel ar +ĠP agination +Ġsound track +Ġpost erior +Ġscr ub +cre ating +- * +ir teen +.d y +.s ymmetric +Ġ"" . +============ === +Ġch assis +ĠnumberOf Rows +Develop er +_b ins +ĠO UR +ri eb +Pro s +Ġwi ÄĻ +" d +Ġasync io +ze igen +_s pi +.A LL +Ġscre ws +Ch inese +Ġapi Key +Ġun successful +ĠSeah awks +OR G +ç« ł +Ġprofession ally +ĠCou pon +åŃĹ æ®µ +Con vention +Ġpol ym +æī ĭ +Ġsalv ation +Ġengine ered +ĠW rest +ĠG CC +Ġwar mer +Layout Constraint +Ġag grav +Script s +vent ure +Ġrefriger ator +Ġinnov ations +ĠRun ner +N IC +ĠRoll ing +Control Events +Ġlo os +p ac +ĉ panel +ef e +ĠBudd ha +------------ --Ċ +åº ĵ +(for Key +Ġl umin +Ġ( ? +ĠA IDS +, user +im ientos +content Type +ant lr +é ¦ +ĠW elt +Produ ction +m ight +ĠV II +", ( +Ġobserv ing +Ġdeliber ate +( control +Ġwith d +Ġsem ana +ST ACK +uch en +N ice +ĠDeutsch land +ĠSpec ifies +d ma +iz io +ĠF acts +_pop up +ĠDirect ors +{ : +[ R +ĠÑį леменÑĤ +Ġpl at +Ġdirect ing +ä¸ ī +ĠGil bert +âĢ¦ .ĊĊ +.q ml +Ġthere after +Ġdis position +d raft +Ġsurge on +ĠIns ider +Bl end +ĠT rev +tr insic +Top ics +rie ve +_FILE NAME +Ġaut res +J ose +Produ cer +er us +Ġpet it +ĠN EXT +ĠF ilters +Ġreplic ate +"] ). +Ġl enders +] ",Ċ +; charset +Cpp Object +Ġfl oral +ĠT ipo +Ġcirc uits +e asy +(& $ +itt a +ery l +_COMM ON +'}} >Ċ +-back ed +(var iable +( Index +Ġvo ir +_loc ations +++) { +ĠLouis ville +Ġgrat itude +.Mock ito +ĠP owers +ie urs +Ġge ographic +ra le +Ġc ra +ĠSp urs +iph ertext +AC ION +- common +Ġvict ories +ĠFinal s +.sh uffle +-m illion +_PRO C +ass ume +Ġil s +DB C +Boot Test +Ġl avor +.test ing +. ast +"] / +m oid +Ġqual ification +ges ch +ĉ put +Ġair ports +J I +Te acher +_un iform +Ġn ama +ĠB ast +ert ype +c apture +get All +ĠReyn olds +oo led +.com ments +Ġch in +). * +Ġи ли +t gl +ud os +Ġd ÃŃas +ch ai +.pro gram +Ġps z +ĉ icon +ph il +ent ral +_WR AP +ov i +Ġnost alg +In finity +ĉy ield +Ġvit amins +Qu aternion +S ink +_g oods +Ġ ........ +ĠW ings +ur idad +-st ory +"] )ĊĊ +idel ity +Type Def +G tk +Ġí Į +_M ain +Ġche z +ĠR aven +Ġpay roll +Ġfreel ance +LL U +ĠM end +ed ay +Api ModelProperty +.Form BorderStyle +Ġeconom ist +stan bul +Ġfre ight +-A gent +(m eta +Ġsym metry +Ġ' .. +.C alendar +- aut +g f +p ent +yc lopedia +Ġwish ing +ĊĊĊĊĊĊĊĊ ĊĊĊĊ +Ġgentle man +Ġê ³ += # +Ġlect ures +âĢľ In +Ġ! _ +Ġh b +ĠV endor +Recent ly +_n otes +æıIJ 示 +" My +Headers Height +_S O +Ġunw illing +Ġsuper hero +g io +ps y +ĠPe er +j avax +& apos +ĠCr isis +ord inal +Mem cpy +++++++++ ++++++++ +- val +Ġwork book +- ap += k +Ġmetal lic +_ peer +By PrimaryKey +_S D +u ator +_SH ADER +) Math +.Trans form +Ġc ows +Ph i +ĠC lem +(_ (" +ĠL ud +-d elay +ĠSec urities +ĠOrth odox +Sym fony +(re port +Ġent ertain +E PS +iz oph +ex ual +IR D +ä» İ +Ġl ith +Ġsanit ize +Ġfemin ine +IS BN +.auth entication +_p ipeline +/ constants +ĠCON F +Ġluc r +ric ia +.t tf +.set Content +Ġst an +ore an +ĠL loyd +.raw Value +Ġg or +ĠBrow ns +Re gression +Ġlower ing +na issance +Ġbl ows +Ġam azed +Ġun related +Re views +Ġrub y +ĠMod ifier +Ġgi ants +. thread +Ġcontain ment +ĠStart Coroutine +um at +ore lease +ĠR andy +@ endif +D igest +Ġsubur ban +=" );Ċ +Ġann once +. variable +\F oundation +Ġa cre +V an +Ġt uples +d ns +ĠStand ing +_l arge +Ġbox ing +Support ActionBar +ĠFort une +ĠR um +_m ultiple +arch ical +Ġf write +_ quote +Ġfool ish +Ġcompr ising +Ġо п +- selected +v f +ma id +N ama +(d atetime +Ġindirect ly +g art +fix tures +ch os +ĠH alo +Ġrec urring +- news +v il +ĠNurs ing +- produ +ĠH Q +\Http Foundation +enc i +au en +Ġv y +ocr acy +Ġdeleg ation +Ġas phalt +Ġset Selected +k ok +/ rest +met ics +ĠNS Date +Ġtravel led +Ġrec ib +Ġm ime +CL IENT +ĠG U +ĠH ANDLE +/ Q +[ z +Ġbother ed +ĠBB Q +ç as +_ex amples +_F IN +Ġwhite Color +Ġastr onom +-d ir +Ġsovere ign +Ġb reeze +Ġin ning +ĠEd monton +g li +.blog spot +js x +Ġvers a +ĠMoh ammed +.J ob +-t oggler +Ġп олÑĮзоваÑĤ +ard on +Ġnew born +Ġnav al +note q +Ġtum blr +Ġh entai +ĠTyp ically +Ġlo ot +.S prite +Fl ight +Ġw avelength +-s k +ĠEl le +_ exports +Ġ Ñı +ĠI H +izoph ren +Ġí ģ +_pr imary +Ġmo is +ĠB N +Ġsystem ic +Ġdifer entes +IN CT +Ġ'' ĊĊ +$ q +Widget Item +cl ide +$ file +L emma +/ table +ag rid +ĠMongo DB +int e +Ġapp rent +ÂŃ ing +.D b +Ġà Ĥ +ham mer +=' ';Ċ +Ġbro kers +it lement +sembl ies +E le +{ x +Ġlast name +< - +Ġfl atten +_b and +.R oot +.read FileSync +==== == +.r x +? čĊ +Ġmetaph or +T i +con te +Ġdeb it +Ġcont empt +Cpp Type +æĶ ¯ +Form Field +r atio +os opher +Ġimpl ant +P URE +Ġal ta +_man agement +Ġref ine +ĠCheck Box +ĠChar l +- version +cond itional +ven ues +Ġrif les +Ġoff spring +Ġmill ing +Ġshar ply +Ġunder water +( origin +_ Control +Ġ. $ +Pl ugins +Ġdry ing +Ġillustr ates +- u +Ġveget arian +n pc +He art +; ',Ċ +com ma +te enth +as an +/s pec +_m oves +-m argin +Ġing en +³³ Âł +Ġpro jet +Ġo tra +Ġbr as +. utc +Ġsle pt += sub +ab ilit +post er +Ġs dk +ounc ill +Ġw d +Pre paredStatement +ĠDr um +( attribute +ĠEther net +ĉ DB +Cal ifornia +c ube +[ I +.C reated +ĠH M +Ġtr acing +Forms Module +- you +.c urrency +feed ing +Ġt body +L i +acc ion +n as +Ġtr ouver +N ONE +"} ,čĊ +Ġf tp +With Identifier +pol ate +File Info +Ġpurs ued +ĠĠĠĠčĊ ĠĠĠĠčĊ +DE SCRIPTION +} */Ċ +From Nib +Ġdecor ative +_S SL +(ch at +T LS +Ġsurpr ises +al culate +ĠS plash +( Configuration +ĠS EM +im son +/lib rary +< Double +. robot +³³³³ ³³³³ +ĠCP F +ĠUnder standing +Ġcos metic +ĠX t +t ips ++ k +(" ' +ĠP DT +W AR +.get Object +ĠTrad itional +.sl ug +ĠDi pl +=" ", +ĠFil ms +ĠAn im +.h elp +Ġemb assy +ĠBoot s +Ġb unk +-r isk +Ġp ci +Ġ/ \. +ĠI PT +Ġcrash ing +Ġip v +_ ke +ĠRES P +.Log Error +Ġinade quate +I on +ĠF ür +ric ula +Ġshould Be +al ready +']." +G ED +fa q +Ġoption ally +_D is +ĠSuccess ful +ĠC ensus +Ġinc arcer +_C ARD +Ġav iation +ĠG ym +Author ity +.B ean +sh ader +Not Exist +_Text Changed +ĠST OP +( team +" H +w g +Ġgr inder +Ġstri pe +Ġpres ervation +Cl aim +avers al +ware house +target s +Tr ust +Ġal lev +, www +ous se +_ch an +_S ize +system s +Ġobj ection +ĠK ane +Ġcor ros +ĠD SL +Ġu a +ĠM H +ĠStrateg ic +_t cp +Ġê° Ĵ +Ġborrow ed +ĠA ch +ĉ command +Ġg ps +le ston +iche ver +ĠU A +Ġassault ed +Ġspecial izes +ĉ search +Hot el +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +ĠP itch +Ġ Ùģ +READ Y +Ġparent al +Ġg éné +Ġdonn ées +Ġdet ain +T ARGET +Ġprotagon ist +Ġclear Interval +ĠIcon Button +ĠGet All +Type Info +E H +âĢľ They +Ġ{ [ +Ġg ag +Ġ Ú© +ĠD ropdown +.f ree +g one +im ens +Ġinst al +ĉc url +_C AN +ĠB one +ï¼ Ķ +ony ms +-g overnment +.binding Navigator +ĠD ans +ĠMc L +( en +>( _ +ÐĴ Ñĭ +.* ;čĊ += j +-c or +S on +.ToolStrip Item +- around +_X ML +end Date +Ġsl ack +Ġrot ated +Ġno qa +Ġc ottage +Ġencontr ar +_s kill +hou ette +! čĊ +. weather +Ġemphas ized +å® ¶ +ĠÑģ пиÑģ +ĠComp iler +( android +ĠâĢ º +. turn +Ġsup pression +_c alls +Ġ* @ +(str len +.h ex +ĠB ills +ĠR SA +Ï Ĥ +ĠEs cape +ement ia +Ġfront end +Ġp int +_ex c +zz o +[ ],Ċ +Ġ"',' " +. Environment +Ġafore mentioned +Ġend ure +prot otype +ther apy +ss i +D eg +_pl ugins +.user Info +Print er +ĠPRO GRAM +Ġru ins +Ġempir ical +Ġcraw l +ĠBo iler +- comment +.sub plot +_ et +Ġ'. ', +min or +ĠCustom s +Ġy aw +under line +ĠCom o +( (' +(m ean +Ġcha que +ĠBlock s +.r ad +ilib rium +Ġweb driver +Ġmel hor +d ana +ĠAb use +ĠSouth west +ĠP aren +PERT IES +ĉ IL +Ġscre am +v u +Ġin comes +Ġn im +Ġl ace +Ġcompens ate +Re verse +D at +_att ack +Ġn our +ach en +ce k +< Func +w ie +com pressed +-m atch +(" ")]Ċ +im ized +. orientation +.compare To +Ġmass aggi +Ġìľ Ħ +Ġel bow +Ġant ioxid +undred s +/ tools +ĠR OW +an mar +ĠW ow +_t icket +Program ming +Ġthe or +-re view +() )));Ċ +ĠRichard son +ĠP ocket +] [] +am pp +_ health +ĠP OP +ĠNav al +Gu ess +Ġancest or +.Get All +.local Scale +ĠM apper +Ġaccum ulation +Ġsim ulated +ĠDr ivers +Ġd és +cur ring +Ġele phant +Ġadvert ised +Ġmail box +SH IFT +ĠMon ica +Ġan c +Ġward robe +Ing redients +Ġ|| čĊ +ipp y +Ġantibiot ics +av ings +(c x +ĠFerr ari +ĠAn imator +.d type +rem oved +order by +Ġc res +oc ê +Ġp ym +ĠCirc ular +@ index +ĠW arm +S ay +ĠAss istance +Ġcur tain +ĠMont e +IL ER +ĠC VE +ĠD uck +ĠAll ows +_f ire +ĠDer by +Ġre pos +Ġhttp Client +Ġpsych iat +Ġnow adays +Ġcaut ious +ĠComput ing +Ġcompletion Handler +ĠWel sh +ĠB EST +Ġstress ful +_P E +æĹ¥ æľŁ +ĠData Frame +ĉ Integer +_P rint +M oves +Ġtransform ing +.B atch +y ahoo +Position s +ze j +Ġno od +io res +_ * +Ġcl k +ĠF loyd +Ġh ap +font size +Ġn az +.not ification +ĠDep ression +Ġac ne +*** ĊĊ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +.cont ents +yn th +ĠStra ight +')}} "> "+ +Ġtoken izer +Ġsovere ignty +ĠP ence +() ");Ċ +Ġpesso as +.G e +ĠIn cluded +Ġpag ina +Ġex posing +е ÑĪ +_SC RIPT +/$ ', +Th umbnail +× Ķ +webElement X +webElementX paths +press ure +ĠCur ry +_C P +OL UTION +ILE S +prot ect +ool a +Work space +{ };Ċ +ĠU NS +Ġsymp athy +ro ker +Ġrem odel +ĉc ell +Ġat op +.Full Name +Ġfa ut +ĠE asily +_d ynamic +Ġfr amed +Ġmot ive +è· ¯ +s am +Ġmar ca +ĠText EditingController +Ġde structor +cre am +Ġr ude +ĠB old +ĠInd igenous +Ġg ens +Ġrel acion +(s ystem +ĠUIF ont +_char ge +UST ER +E V +.N amespace +Ġmer ger +Ġcal loc +g ang +Bad Request +Ġs per +-d esign +Ġâ ĩ +Ch an +Ġorgan ism +, ) += id +_pl ane +ĠC ases +elf ast +ĠLegisl ature +ĠF aker +Ġinv oking +- utils +(). ' +.f ace +Ġguard ian +my Modal +Ġclip board +ĠAT M +Ġpe as +ĠS ylv +.c alc +ĠContact s +int Value +Ġmodify ing +ĠBar b +. loss +_per centage +Ask ed +(l st +ategor ical +- files +ĠRoman ia +.A c +Ġh ai +ĠF lying +Ġ ż +j p +ĠTr ainer +. arc +_de g +Ġtrace back +Or Fail +F LOW +. old +oy a +g mt +is empty +Ġvacc ination +Ġob solete +recogn ized +Ġru ined +ĠRe in +ĠTr acking +xf b +ا ÛĮ +Ġvæ re +Ġbr yster +ĠIT S +Ġdest iny +Ġsw ear +Ġred es +Ġcl f +Ġfl ipped +ĉ head +Bl uetooth +ĠOver rides +: Boolean +_ = +_l r +sp awn +: index +VAL UES +is key +? ");Ċ +.syn thetic +ĠCheck ing +struct ures +ip ing +Ġvoc als +- Up +ĠManufact urers +ĠMar riage +代 çłģ +Ġgar ner +_C lient +par allel +RI END +Ġvine gar +seg ue +J B +Ġcontact ing +ĠCar roll +Ġout reach +t ensor +_var iant +Ġthe at +lic able +{ | +t iny +_ letter +Ġp encil +HeadersHeight SizeMode +ilt ro +.auto configure +.d rag +.use State +ĠB MI +h int +Com pile +* \ +en ary +Ġl vl +.C ache ++ =" +_t v +ruit ment +Ġf read +Art icles +f ila +Ġpack aged +âĺ Ĩ +AT HER +ĠPl anned +s cheme +Ġdi ary +Ġoff enses +/ F +ĠSt ick +Ġc erc +ĠS lee +ĉĉ ĠĠĠĠĠĠĠĠ +< Image +Ġè® ¾ +- editor +pie ces +ĠD rama +Ġ// //////////////// +ĠT asks +AR C +g ateway +.get cwd +.M etadata +Ġguess ing +åľ° åĿĢ +Ġsm arter +ĠGet Enumerator +Ġe fter +/ operators +ĠGL float +Ġf ør +Ġop aque +ä¿Ŀ åŃĺ +Sp read +SY STEM +Ġinv ersion +ĠBasket ball +Ġsim ulations +Ġden ies +Ġa vez +_list ener +Ġenh ancing +ĠMy th +ĠL akers +_M D +Nd Ex +D ATABASE +Ġt á» +ar th +[ left +Ġcontest s +st ile +(K ERN +_f c +_p m +Ġpres idents +Ġhospital ity +Ġfade In +RO PERTY +_m aps +ĠDefinition s +Ġassess ing +Ġus ar +Ġquant itative +mo z +Be autiful +[ (( +b ons +f requency +Cont ain +Ġpuzz les +ĠCast ro +Ġv illa +Ġkind ly +Font Awesome +ern a +epoch s +_dat as +ĉ ip +.p adding +ĠCont est +Ġed itions +Ġdispro portion +ĠI CO +Ġcome back += value +ri ad +-s ort +Sub mitted +(n etwork +ĠC el +Ġinstall ment +l ashes +.List View +ĠV atican +(Media Type +IV ED +reach able +: Is +ĠC ITY +äº ¬ +ĠHelp ful +Ġba ÅŁ +% čĊ +Ġpsych iatric +Ġrec ycled +FORM AT +ĠG row +b ine +G it +.s s +ĠWe apons +ĠSt y +_ arrow +* self +ire ment +Ġdeg li +App Delegate +_b anner +Ġcoordin ated +ĠWeb cam +Ġcelebr ations +. act +******************************** **************** +( show +Ġweek day +Ġconc erts +ол н +cl in +Ġcr on +ĠN im +.set Vertical +ĠEll en +س ت +ĠS AM +E ff +g z +ste am +Ġant ique +ph ysical +ĠForm Data +.set ter +ĠPO INT +B on +Ġflav our +erv ention +_ENT ITY +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠ +Ġintr insic +Ġæ İ +append To +aram el +) ]) +ĠRecomm end +) m +OutOf Range +Ġkn ight +Ġsat ellites +ĠTit ans +Ġweigh ed +ĠD ana +e ase +Ġs ip +S IM +ĠDevelop ers +mal ink +/ check +_P LL +n ung +Ġdry er += A +.d w +_S QL +Ġsub plot +D ROP +Ġprot otypes +Ġhour ly +display Name +Ġas i +ĠViol ence +Ġastr onaut +Ġdat atype +Ġinformation al +Ġinvestig ative +etermin ed +ren al +; '> +ĉc ol +V G +_ boolean +re cent +Ġ* )ĊĊ +ĠRain bow +om men +Ġl ur +Ġopp ression +(", ");Ċ +ĠFac ility +DEF INED +Ġne on +Ġoff ender +AF P +ĠClean ing +[] ): +Ġund ocumented +.Re positories +ĠG uitar +аÑģÑģ ив +Sk ills +Ġtestim on +rypt ography +ĠAm ber +ĠSt alin +Ġl one +Ġap enas +Ġdies es +ĠAr duino +è½ ¬ +== - +_A ct +Ġc oded +âĸ ł +amb urger +-link s +Ġarm our +.H igh +get Content +st ag +Ġhe ck +ĠìĹ Ĩ +ĠMc Connell +ĠCon cert +ĠAl loc +ä re +.replace All +Ġpart itions +rot t +ĠF le +_T REE +reason able +ĠReport ing +Ġbillion aire +s cores +min s +- eye +M ORE +ab ort +ĠSW T +Ġin verted +ĠTe achers +; n +Ġast ro +н ов +ани ÑĨ +product o +c ountries +ĠO wen +Ġcont amination +Ġv ibe +ĠEll i +.s cript +ĠOl ive +D MA +v ier +: semicolon +-m odule +gress ive +ag u +_ players +Ġresult ados +start ed +scroll Top +==== = +Ġweigh ing +Ġ[[ [ +z ahl +( NS +ĠAssert ion +le ague +.setText Color +ĉ Message +Ġmom s +_A F +. wh +AL S +Ġaut re +] ĊĊĊĊ +.op acity +ĠBudd hist +Ġde af +ĠOrgan isation +(G lobal +ens ch +Ġhead ache +ĠAli en +_in ode +ĠSt ark +Ġæ ī +-l nd +ore f +_fe at +Ġpedest rian +Ġnom inal +Ġbal loon +Ġspr ites +Prototype Of +ĠA post +ĠF EATURE +O H +Ġre cess +ĠDon na +con sumer +$ GLOBALS +ĠG IF +- frame +In icio +Ġpass ages +Date String +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +.by te +B ug +initial izer +p kt +od ium +ĠD ER +. ops +ler i +Ġgift ed +Ġdet ach +ter rain +elt ers +ãģ ı +. loader +ĠN GO +str ncmp +K h +(font Size +ro cket +Ġpreced ent +ĠAur ora +ĠEx periment +is phere +Enc oded +ĠâĢĵ ĊĊ +Ġpy ramid +ĠAnn iversary +of il +ë Ł +( plugin +C oeff +Ġcooper ate +Ġpredomin antly +IS M +Ph rase +_DEF INE +Fl ip +AMIL Y +ĠMark ets +ĠStream Reader +ĠComb ine +Ġmanus cript +z za +, tp +Wh atever +IT ICAL +ighb our +Data Provider +.Text ure +priv acy +.S DK +Ġre charge +Ġc pp +ĠC FG +(h older +(p y +m ot +Ġsav oir +ĠR osa +ĠPC s +Ġí Ļ +.her oku +Ġf ren +ĠR iley +ag ate +Ġs ond +.x lsx +Ġh acked +st ad +G i +Ġsan ity +ĠSql DataAdapter +... ", +ĠP ussy +Ġ **************** +Ġhass le +_P ARENT +ĠU AE +Ġbegin ners +( Client +Ġstatist ically +.h our +ed elta +Ġtr action +uel ve +ar at +Ġsa una +IN VALID +Ġindict ment +AL LE +Ġdiss ent +ĠTyp ography +Ġintention al +s it +ĠAn imals +Ġcoun tryside +Ġu art +} \" +Ġseam less +¾ 示 +Ġaut os +Ġ"' ";Ċ +Fl ush +ANN OT +Ġal gebra +ass oc +ĠW aters +Ġprepar ations +ron ym +[, ] +S ans +Ġarm ies +ipe g +Ġcream y +. art +et re +ĠAn imated +Ġun pleasant +eme an +g reat +i Äħ +ĠEar lier +Ġch ic +Ġpres erving +(ex ec +ĠInvest igation +ĉG PIO +Ġrig orous +ij o += num +Ġtool Strip +) set ++" & +ĠAcc eler +Ġdevelopment al +is posable +Ġflaw ed +re ne +Up dating +Ġwatch dog +Ġden ominator +Ġsubur bs +Ġ... ) +Ġconv ictions +c losure +.I P +Ġtransl ates +.sw t +.Tr ace +Ġmet tre +.is Enabled +ĠEffect ive +.to Int +Ġen chant +Ġst unned +Ġpo i +/ code +ad m +.datab inding +ĠL orem +________________________________ ________________________________ +Ġled ger +Ġcar a +ĠG ir +Ġwa its +Un o +Ġc wd +è¾ ij +ĠT Result +Ġre jo +Ġem itted +ĠWest minster +ä¸Ģ 个 +ne k +_T is +Ġen act +ĉ with +org ia +Ġj ue +Per form +SP ATH +.top ic +ĠD aten +Ạ§ +Ġsit io +_M M +" So +b ial +Ġsc oped +Re quires +ĠT OTAL +ĠCh ancellor +( contents +Ġste alth +dev ices +-p ass +ili h +ĠMal colm +ĠDep ot +Ġconfig ur +a ussian +_con straint +в еÑĤ +G RA +ĠR ates +.dataGridView TextBoxColumn +ĠNob el +it ics +Ġignor ant +ĠReport er +ĠEb ola +ĠSh ock +_re lation +ĠNin ja +) c +Ġt icker +.is Checked +ĠSup pliers +ĠRap id +Level s +âĤ¬ âĦ¢ +ĉ queue +Ġch op +ĠUn ix +re ject +-c alendar +(s ort +è ne +erc icio +Ġh ect +CALL TYPE +rou pon +Ġrent als +auth ors +{ name +ĠF IFO +Ġl assen +ĠN ous +Ġsn apped +Ġfert ility +" log +click ed +Ġplant ing +Ġg b +/ output +PE AT +Ġc ategoria +Ġb ach +Prof essor +in th +"] čĊ +Rec order +ser de +ĠTrans mission +tr ad +Ġtur bo +_VER TEX +\ Event +il ver +Ġbod ily +ĠS ources +Ġkill ings +.xr TableCell +Ġfold ed +/ legal +un er +ĠR ifle +ĠM IDI +_Selected IndexChanged +.Size Type +ĠWeb Socket +Ġsele ccion +S and +ot ros +Ġenv ision +/ etc +ĠMel issa +Sp ot +но е +_ ARM +At tempt +ĠB I +ãģ Ķ +ĠD U +Ġback lash +str ide +/ classes +Ġtext Color +_st aff +ob lin +agent a +.c ollections +ill age +' čĊčĊ +fl atten +_s ales +_M ASTER +T W +_d a +P itch +ph ies +Ġz ombies +ĠV ERY +ĠPharm acy +Ġprogress Bar +Ġhas htag +S idebar +@ stop +(p c +ол ж +MA KE +ĠCor on +Ġkv inner +ĠM aid +b ob +.title Label +Ġsuccess es +ĠDemocr acy +ĠSurg ery +Ġcou gar +Ġcur so +Ġl oro +ist ency +Sen ior +æ k +ĠA AA +ĠBO OK +к о +W STR +Ġ*/ ,Ċ +oy al +.v ector +ĠS PEC +SS F +Ġcomp uls +ĠAppe als +ĠW inston +ĠMock ito +con trib +. available +entity Manager +ari as +_s ale +_r s +Ġdec oding +Ġloc ator +ol ith +Ġk ol +Ġasc ii +ĠR ut +/ interface +ĉĉĉĉĉĉ ĠĠĠ +ĠN umer +.fl ip +-d el +Ġbol ster +on omic +Ġz m +L G +Find By +Ġadapt ive +lo o +Ġv ue +(re verse +_c anvas +. roles +ific ado +ven ient +" As +ĠEn tr +al igned +Ġbere its +/// ĊĊ +.g wt +. employee +_cl i +Ġanticip ate +éĻ IJ +Ġp ik +Ġmush rooms +(t t +Ġo ma +ĠSan chez +_g oogle +. Valid +ĠFile Name +iv ative +k ed +-w ar +Ġm aturity +и д +Ġmin er +Reduc ers +ĠLat Lng +_ST D +D igits +Cal c +-up load +Ġhand ic +ี à¹Ī +egr ated +ĠST M +C lients +ĠTur bo +SY NC +Ġphotograph ers +. Out +.char acter +B UILD +.un lock +Ġar ises +ĠCommand s +(" ");čĊ +_F ORE +; ', ++" ' +. Images +") { +ĠM eyer +Ġneg atively +ĠD LL +Ġex e +Ġdef iciency +Ġwild ly +-s witch +con struction +Ġexception ally +ĠL iz +/j ava +Ġtheir s +ĠCont emporary +l is +.fill Rect +ĠN FC +Ġre he +(num bers +Ġr aster +Ġfig uring +Ġshow c +ĠJ ill +Ġarc ade +ĠConstruct s +md l +(' | +Ġident ifiers +Ġst ellar +( Connection +Ġ" {{ +y or +(m ysqli +Ġdo ve +Of Birth +.dis connect +_h i +Ġzw ischen +ĠGr und +i ros +_A rray +.on click +ans om +An swers +ĉ remove +F a +Ġhur ry +-in f +Ġget Class +ĠReg ulation +ĠFLAG S +m isc +K en +_ heading +G Hz +- entry +Ġbi ography +S ig +-m f +Watch er +âĢľ A +} px +Ġsp icy +_s q +L ost +(tr ack +а ли +Desc ending +< bits +qu ine +ĠAdv oc +_S N +ĠHann ah +PO P +Ġem itter +Ġc yn +ĠC AD +? ). +/ set +ĠS ister +ĠEnd point +Ġmen or +Ġinter p +r k +id le +Ġout fits +. vertex +Ġc lic +ARE N +Ġpost ure +ĠOpport unity +v x +ĠFor bes +.D irection +Ġres ide +Ġremember ing +nest y +Auto resizing +pro viders +ĠA H +Ġhur ting +ĠL ily +eval uate +lij k +p apers +ĠSm ash +ĠL AST +Ġwell s +w asher +_RO LE +ĠD anger +* (( +_re pository +ĠRes olve +ĠRoom s +_R G +ĠQ T +o op +ĠHe ap +Ġslow ing +Ġgrat uite +_c atalog +Ġpol ynomial +L y +pc s +F ox +ĠC yr +Ġdim in +/ month +S alt +Ġh ind +.P ER +For um +c en +_p ol +íĺ ¸ +Ġin ser +( ~ +@ test +ĠGold man +Ġupload ing +F c +Ġkom mer +Ġm itt +_log ged +Ġbu cks +-l ayer +) };Ċ +ĠO M +Ġv eg +col our +Ġоб ÑĬ +Std String +_ que +ĠT ian +Ġspecial ize +и п +Ġк л +tr ial +- edge +Ġm ars +OG LE +Ġempath y +ĠB om +Ġcoll isions +Ġcart e +ĠTe il +ĠM PL +Ġporn ô +Ġa irlines +A ws +N s +ĠSp awn +( use +é» ĺ认 +Ġy acc +st or +Ġconf ess +Ġpe que +r age +? "Ċ +/dat atables +ĠSh ower +__ / +Ġcryst als +Ġbus car +ĠH aus +iz ação +_ entities +ķ Į +ļ Į +x cc +v irt +-che vron +( Result +c ake +COM E +Ġprohib it +ĠCh ess +Ġbe aucoup +ĠÑĩ ÑĤо +R UN +ĠI K +ó ÅĤ +_ Update +Ġsle ek +ĠSpec ify +_c redentials +ÅŁ t +ĠUser Name +ĉ Value +Ġarray List +Ġex changed +ips is +.re lated +ĠSe ite +_B AR +ĠL em +ĠW ATCH +ĠC lients +Ġ. * +ĠEar l +-re port +Ġforeign ers +Ġstrengthen ing +ĉ Description +(g o +.tool bar +Ġcalcul ates +ĉs ource +Ġcz as +Ġre cl +ab o +Ġlocal host +Ġ^ {Ċ +.P op +ĠDes igned +\ Abstract +H old +ĠGuid elines +ipl ine +Ġc aching +.Re ader +_ext ernal +.str ptime +ĠWeek end +-M ar +ĠBe i +Ġ{* } +ĠR ud +Ġexpl or +ĠBou levard +C ash +Ġprep ares +Ġserial ization +ew ater +Ġad c +: ĊĊĊĊĊĊ +Re fer +Ġsc anned +} }ĊĊ +ĠF ul +Ġtour ing +ãĥĥ ãĤ¯ +> (( +sur vey +Ġí ĺ +... ')Ċ +ĠDiv ider +os l +_C ANCEL +_pre pare +st in +ĠHe ath +.Primary Key +ĠâĨ IJ +ĠLocal DateTime +Ġcooper ative +L earning +.en queue +Ġgo og +ĠReg ression +im ates +Ġvoy eur +ĠDr ink +pl ug +Ġl ender +man a +Ġperson nes +yp se +Ġun link +ĠRav ens +Ġhur d +Ġperiod ically +ARG S +ĠG H +char acters +... "ĊĊ +- establish +Ġd n +( condition +ĠGr avity +Ġest as +_f ocus +Creat ure +(s ite +Ġc arr +ĠR L +ĠR I +ĠM oto +AS F +ĠLuck ily +ĉ Route +Ġent ropy +(" ," +Col lect +( contact +ĠFlo rence +Ġpremium s +Ġlif ecycle +Ġb ans +x ef +Web Kit +ĠFlo ating +Ġcos a +Spec ific +ĠLo ans +b read +Ġdes criptors +Ġ{ :. +TH READ +ĠT rent +Ġsc op +Q A +ĠAnt ar +p el +_d ifference +_ch anges +(... ) +ĠR otation +ĠLG PL +ĠJ UST +(T ask +_sub set +ĠTR ANS +åĬ Ľ +ĠSc out +-p opup +Ġsm oked +_C lass +Ġturn over +br akk +ĠRock y +t as +.Regular Expressions +ĠElli ott +ĠSp inner +DU CTION +Ġlib re +Ġmol to +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠ +ĠF TP +m peg +(f eatures +Ġb ald +ĠV id +Ġsh outing +L int +Ġsock ets +Ġpro w +Ġnouvel le +isc ard +ĠS ponsor +Ġconsult a +)) ); +Ind ian +ĠR aspberry +Ġteam mate +ĠJ WT +ĠGh ana +Ġc akes +pr imer +form a +erg arten +_M anager +Ġpre season +G AME +| " +ĠBro ck +Ġoccup y +Ġdecor ations +á nd +Ġc ot +Ġpar an +D isk +rem ain +> ? +Str ong +Ġfr ance +ĠE ra +-c r +.Buffer edReader +ĠParad ise +ĠV AT +ĠAnd ers +Ġlim b +amp oo +Ġimper ative +UT ILITY +ĠRec ognition +Ġragaz ze +Ġpop s +yp ress +Ġemb argo +// {Ċ +Ġsy ll +P TR +åŃĺ åľ¨ +Ġdid nt +Mail er +Ġacad emics +ĠFra uen +ne ider +- rel +Ġrain bow +( In +Ġslic ed +============ =Ċ +(s end +NSMutable Dictionary +v os +(p ackage +Ġord inance +view er +ĠSant os +-s elling +Ġgo v +ett le +Ġfound ers +Ġw aking +sl ashes +-p ound +re cht +ا ت +.on Click +Ġn ord +st änd +_ when +UT ERS +ic c +Ġcaps ule +ĠW id +M arc +ภ¸ +ro red +UG E +LO UD +ĠAud it +ip ients +op ian +ĠS ue +Ġwur den +.H elpers +Ġf actions +[ np +-th an +Ġre co +Ġk as +Ġcmd s +/n etwork +xb f +get Color +Ġbi ased +ĠL ak +D atas +vent s +Ġë ² +_P S +. Validate +Inv oker +Ġne uen +Ġju venile +V ISION +Ġdev ote +Ġlin ha +Ġdiscount ed +\ Config +Ġworth while +Ġskin ny +ĠC ourses +le ys +ĠMort gage +K evin +Ġannounc es +]) * +res ervation +Ġæķ ° +Ġprejud ice +ĠString Comparison +Ġbe ard +-w in +ĠS ão +ĉ ms +j al +ĠE arn +_ ports +ĠN ombre +_C OR +ĠB UILD +.s ound +Y ellow +Ġlineback er +Ġchar itable +j ug +_NON NULL +ĠD ental +"> ${ +ĉm atch +R ussian +Ġvers ch +Ġp inned +Ġadopt ing +Options Menu +P ag +Ġpair ing +Ġt read +erc ises +ĠSp read +) i +ĠB AD +_t f +UI ImageView +pop ulate +b ab +ĠÏ ĥ +[ ++ +Ġopi oid +Ġ## Ċ +d type +ĠStart s +('/ ') +Ġperson als +-mark et +Ġredund ant +ĠEss ential +Ġscrap y +Ġи м +a cl +Ġcre ar +ĠB end +Ġrel ieve +- room +w ife +Ġv Ãł +ĠQ Point +Ġqu asi +Ġmethod Name +\x c +ĠPer u +/ The +. orm +Ġv iz +/p df +Loc ated +Ġconfront ation +ĠChampionship s +Ġhyp ert +Ġd j +ĠUser Info +ĠåĪ Ľå»º +\x b +(s im +Ġ== Ċ +Ġst aging +Ġdr astically +åŃ ¦ +l ords +. less +вед иÑĤе +ĠB ucket +ĠM am +. term +_p i +c zy +.p ub +prec io +ĠV irt +Ġrom an +it at +L ex +_inf os +Ä ° +. other +VE LO +Ġp onder +Ġh anno +( Page +do i +Ġpol ite +Ġprogram mer +D ies +$ d +Ġrep lication +add Column +fr ican +Ġl eng +be er +o it +Ġw asting +yl im +me asure +N eg +Ġpart ie +.con sole +ĠGu inea +TE L +_f act +.ch unk +Ġl ent +Ġall er +Ġठķ +_id le +Ġad missions +JSON Array +Ġv ibration +.h elpers +å¤ ĸ +Ġh en +j ohn +Ġì ĥĿ +Ġjud gement +Ġge en +ter ra +^ { +ĠI z +Ġc â +inst ances +Ġthreat ens +Ġm üssen +Kind OfClass +Ġstoryt elling +_d emo +ri as +Priv acy +h ift +ĠY i +es or +íķ ł +ens itivity +.W riter +ภĤ +D istrict +.get JSONObject +Im pro +(get Resources +ĠS PELL +rodu ce +Ġslow ed +Ġlin ewidth +Ġhonest y +ĠCo ord +ĠF ork +ĠDispatch Queue +ĠCl iff +ĠW iring +_TIM ESTAMP +oll ah +av oid +++ ];Ċ +sem antic +-c ss +Ġv eto +ĠM err +Ġlegisl ators +CEE DED +Ġquestion naire +ĠP ills +Cal culate +(c ore +' e +Ġdis like +ĠPre ferences +_EX TERNAL +è° ĥ +Ġd odge +æľį åĬ¡ +.n ames +.draw Image +_p rom +uck land +Ġ<$ > +ı z +/s ite +é¡ ¹ +rop he +Ġcomp elled +Ġl aptops +Ġun i +C LOSE +Ġcasual ties +ĠUn iform +Term inal +. "," +D AT +(T reeNode +ĠGand hi +(st mt +AX B +* M +Ġumb rella +an imal +Ġgr pc +Ġwhere by +Ġfloat s +ĉ arg +Ġdb g +Ġexceed ing +Event Type +.SaveChanges Async +Ġ{ {{ +Ġow ed +ahren heit +Ġì § +Ġequ ipo +ur ai +Ġid ol +] ")Ċ +_m ajor +Ġentire ty +inger print +ç os +/ account +ĉ right +urs os +ĠE DT +_INS ERT +Ġsh ining +Ġ< : +Edge Insets +Ġcolon ies +. IM +ĉĠ ĉ +RO AD +CC CC +pl acing +Ġget Activity +em acs +' %( +.click ed +ĠTh em +is ia +Bus car +.re name +Ġo ath +Ġafter ward +ĠU FO +AP S +ĠJackson ville +.s ome +Conf irmed +.s can +ig Integer +Decor ator +sh ield +ress ive +.d id +请 è¾ĵåħ¥ +Ġsh utter +D am +Ġparent ing +ey ed +$ item +-de velop +Ġextract s +Ġdecentral ized +ĠEl sa +_sp in +]) + +-in itial +Ġmult itude +Ġsens ory +ĠMODE L +Ġsafeg uard +ì ¹ +Ġhunt ers +ĠT iny +IN O +decor ate +ĠNo Such +H o +( Response +Ġr uler +ĉ short +Ġc aster +Ġclient Id +Ġp db +ëı Ħ +it ic +ĠGame State +Ġnew Item +)ĊĊ ĊĊĊĊ +ou is +n oc +.BL ACK +_V ECTOR +---------- (); +.get P +any e +Ġneur on +if old +ĠK nown +Bit coin +Any way +ay ette +Ġ' [' +Ãł nh +m gr +Ġcor related +Ġn ause +Ġment ality +has Many +ĠF G +amp ie +IT U +F s +.S p +_b etween +Dep endencies +ou g +Place holder += text +ĠMan aging +ocal ypse +åĮ Ĺ +_m ag +f ld +â ij +C AM +ĠHelp ers +Ġd ost +/ out +Ġassass ination +.get Image +ĠKenn y +.' )ĊĊ +){ // +ĠR anger +Ġg ek +Ġsinc ere +< Value +ĠD OT +ĠVict ory +Ġleg ends +Ġpr isons +(ex pression +ĠR abbit +_s entence +Ġbit es +Ġon Failure +ĠâĪ Ī +K im +.g ender +ĠÎ » +Ġ[ . +"] ); +land ing +-d igit +TE MP +ĉ entry +Ġstrt ok +Ġdesc endants +um no +Ġlean ing +Ġspecific s +q n +ĠSp art +Ġpor r +EDIATE K +Ġse per +' aut +ĠSTE P +ĠBorder Layout +Ġret ros +ĠSalv ador +ĠEN GINE +x dc +T weet +v k +Ġì ² +] << +het ics +c oding +Re ach +.re q +gu ide +.s cope +sh irt +rog ate +SET TING +ĠProte in +Ġe ing +. EMPTY +.d f +Ġclear er +Ġc rossover +ĠTo ys +Ġco ated +.M onth +ĠAtt ach +/ run +.t abs +Ġogs Ã¥ +B rown +.D ATE +Ġf os +åŃŠ符 +W ood +-th ree +her ited +Ġ rop +( ac +Ġembod iment +ĠKenn eth +Ġcan non +Ġb idding +čĊ +.get Resources +Ġl ump +_const s +( ext +ĉd ir +â Ŀ +Ġpadding Top +Ġobs ession +Ġb anning +ĠApp Module +Ġpart isan +Ġcatalog ue +Ġmin ors +Ġpitch es +we ep +Ġundert ake +Ġthem ed +aud it +.scroll Top +Ġr er +Ġsympt om +Ġopen ings +.block s +open id +Ġas sh +-s ave +ĠP ig +Ġreg ain +Ġin icial +/f avicon +ĉ exp +Ġsp ices +isk a +claim s +m ak +definition s +Ġcorrespond ent +ĠCann abis +__ ,Ċ +ĠL ucky +ĠGa ussian +ĠN early +C AD +'] ]Ċ +Ġadequ ately +ĠT ITLE +constitution al +-m m +_ override +Ġbl as +.ready State +Ġremin is +Ġrein forced +ĠColl abor +Ġdecor ating +Ġb achelor +ERRU PT +Ġup right +ip ation +ĠNob le +Ġvalue ForKey +Ġset Loading +.I gnore +å ģ +G lobals +ĠM ent +AS SES +Ġlim bs +ĠH UD +inc i +. iv +ĠQ ModelIndex +F use +Ġped al +_F REQ +( verbose +Ġlong itud +ĠChar ter +ê ·¸ +Ġbund les +. ignore +um bo +EM A +.... ... +s x +.C ard +Ġhe ute +Ġste er +j umlah +Ġ{ _ +_Check ed +Ġf ax +ĠG ust +itch ens +Ġ ))ĊĊ +Ġremark ably +/ XML +- remove +_b t +Ġinc ub +.p ackage +.current Thread +ĠHigh lander +.s ide +s plash +Ġ ici += D +Ġp uck +Ġball ots +Ġhug ely +co eff +Ġp Data +.C OLUMN +ĠHe aling +Ġord in +! ), +Ġ' ',čĊ +(m d +ĠS ask +< strong +Ġsurviv or +.s eries +Ġcaffe ine +Ġ` ( +.TRA ILING +_ Input +(" ^ +z d +& );Ċ +ĠP ing +Ġv oucher +.r ating +-sh irts +ĠRetrie ves +.al ibaba +Or acle +_MO V +Old Data +Ġ/* čĊ +Ġg boolean +Ġ=> čĊ +Ġr á +Ġbl unt +ĠImage Icon +if ik +RT C +Ġfib ers +Ġto ile +.s ent +ĠPy Qt +$ app +Ġmed io +Ġgrant ing +Ġtsl int +ĠM ö +(fig size +Ġhur ricane +Ġlif es +Ġà Ħ +rocess ing +_st andard +- option +')) ) +Ġvac ant +å· ¥ +ĠH ollow +handle Change +Ġdiv ider +ĠEngine ers +Ġsv ens +Ġcompl iant +t anggal +ĠC redits +ĠEm irates +Rule Context +Ġreal ization +Ġdistr acted +]+ = +Ġaug ment +ĠD w +ot p +or rent +Edit ar +.st ock +St udy +pe ctions +ĠGame Manager += cut +Ġf lock +ĠRom ans +th em +-h op +Ġscreens hots +Ġ/* !Ċ +Ġconvers ions +Ġnormal ization +(config uration +Ġa eros +_se curity +! 'Ċ +B onus +ĠDR IVER +ĉ Date +t ie +ĠWy oming +St and +it re +Ġsh oppers +Ġdisadv antage +Ġlik ing +ç¬ ij +Ġunderstand able +SE E +Ġh oy +Ġnin ete +Ġcon fer +Ġnow rap +ĠV ern +, čĊčĊ +imest ep +Layout Manager +à · +ĉw ait +PLE TED +J apan +Ġindu ce +Ġå ¯ +оз в +_END POINT +.h orizontal +Ġacceler ated +rim on +IV ES +Trans actions +Le an +ĠSO UR +wh ether +y g +Ġo id +ĠEntity Manager +OUN TRY +Ġfil a +OLUM NS +IN UE +ĠAn chor +TR AN +wo o +block quote +ĠN urse +ĠCar p +Ġrede em +. try +ĠJ P +Ġtimestamp s +Ġ?> ">< +ĠREM OVE +ĠStar bucks +Re ally +Ġflood ed +.C allback +Drop Down +ip ro +Ġt ended +l te +Ġproport ions +- te +ĠR ena +lic ate +for ces +.ex tra +.auth enticate +в од +¡ ° +Ġfor ControlEvents +Ġsen ha +Ġke in +Ġmin ist +ĠPre ference +ĠTele graph +Ñĥ п +str pos +Ġillness es +Ġp igs +Ġget Intent +S ol +Ġ ¡ +(c pu +[ prop +s creens +'); ?> +ĠAct s +Ġstr dup +Ġaver ages +an al +ĠCas ual +Group Box +ĠHand book +/ comments +Ġnumber ed +Ġbroadcast ing +çĽ ij +.native Element +.m u +Ġupdated At +ĠDoes n +.A C +.c oll +Ġrec order +_sh a +B g +b il +Ġbol ts +Ġç ¬ +Ġim posing +ĠInformation en +_flash data +e conomic +Rem ark +uc as +ĠOff icers +ĠT ER +W alk +Ġmerc ado +_g enerate +H Y +Call ing +s nap +script Id +. operation +ĠFl ame +l iness +Ġrent ed +_t oggle +-ch anging +ĠT Y +' util +EE P +Ġgraph ql +ĠUn i +Ġimp ulse +.B asic +Ġenerg ies +M ARY +ĠMar cel +Ġmort al +Ġf res +m ens +m otion +Ġsample d +âĢľ That +id ay +qu ipment +get Int +ĠA bsolute +,' " +un ed +.sh are +Ġ} )( +mm m +ĠR ising +ä» » +Ġun employed +x fa +.f ollow +ĉĉĉĉ ĠĠĠĠĠĠ +sl t +.P hone +Ġkn ives +Ġe ve +on Click +] ))čĊ +ĠW itness +ĉ NS +ĠE OS +ĠSte fan +ĠPri est +âĢĶ which +Get String +. By +Ġup stairs +Ġdetr iment +bro ken +emb ro +Ġnic otine +il ion +Ġaston ishing +_ aff +ĠLess on +Ġaccident al +od or +Ġdec ir +Ġnew Name ++ . +çĽ ¸ +igs list +ĠG ithub +Ġsuccess ive +rac ial +Ġen viron +éªĮ è¯ģ +Ġredirect ed +T OTAL +Ġgrab bing +ĠL ance +Ġfor fe +_C B +å¾ ® +El apsed +_w ay +(Dialog Interface +_me asure +x bb +D og +Dep art +-s rc +res olver +with standing +_sh ell +ĠLast Name +ĠAv iation +Ġbegin ner +("% . +(to ol +Ġн ов +: init +(A PI +ĠMorr ison +vt Color +Ġstap le +/ INFO +Ġsupern atural +Ġste ak +tim eline +zz le +" `ĊĊ +Second ary +ĠNep al +.String Utils +Ġad am +Ġ( ... +Ġsub stitution +Ġboard ing +ĠKey word +ĠAss ault +dbc Template +Ġorder Id +( engine +.assert That +ĠVen us +Ġhomic ide +ĠA val +Ġg utter +ĠSupport ed +/p art +Ġac claimed +H istor +Ġmes es +ü ber +ĠRen ew +Ġgr as +ĠE k +Ġin file +ind y +.m usic +.S croll +ĠA ges +ĠNar uto +ĠG ather +Ġconfirm ing += (" +Ġpitch ed +ole y +Fr ance ++' " +$ total +Ġon de +Ġd itch +_s igma +Ġcontinu ity +re ward +- load +Ġproces o +Lock ed +st aw +Ġsp inal +l azy +! == +j est +Ġd un +ĠRod gers +ĉ grid +Ġlog os +ĠBeng al +.s uper +Provid es +Ġnut rient +.T imestamp +IZ ATION +åĨ Į +Ġf ats +ĠX xx +ct ica +Target s +Ġcont ours +Ġre ordered +: Array +Ġtoler ate +V ir +Ġter ribly +Ġbr icks +(& _ +h b +Port al +ĠB read +. which +ÂŃ t +as InstanceOf +Ġj object +ĉ length +_M T +; ">čĊ +_EX IST +Ġmat ernal +RE L +Ġê²½ ìļ° +he e +Ġlayout s +ĠL ap +ais y +Ġst umbled +ĠU IG +ĠS co +Ġimp aired +RES SED +Ġab uses +V F +AR B +.N AME +r ch +prim ir +_com pleted +Ġp enny +Ch rome +(b egin +ern en +- checkbox +Plain OldData +ĠL PC +r ade +sp ir +Ġcon ceived +T ips +ĠIo T +ĠG an +èģ Ķ +Ġbi ases +Ġconsult ants +ple d +_ ht +associ ated +], ĊĊ +Ġdelight ful +ĠÑĤ ек +Hel vetica +( load +-exp and +_W IDGET +to a +ĠA kt +Ġom n +Ġcl auses +Int el +*/ }Ċ +_reg istration +Ġold Value +Ġrest oring +Ġun real +O VER +ĉĊĉĊ ĉĊ +AT S +_pro be +Ġdiv isor +.update Dynamic +å¹ ³ +Produ ces +st amp +.j boss +ĉt ask +! (: +Ġpsych ic +@ class +M artin +ĠPass ed +clar ations +h el +а Ñĩ +ĉc opy +-b in +z an +ig ram +া ঠ+(s ig +ĠC aval +_ ## +Ġ% = +out lined +ĠAc id +Ġunpredict able +-d ashboard +Hex String ++ c +.P ublic +Ạ© +Ġconvey or +ĠE B +Ġselect s +Ġknock ing +ĠC ec +IBUT ES +owa Äĩ +g atsby +* v +ent ropy +Ġdispatch ed +Ġcam el +ĠSat urn +Ġover weight +( phone +par able +% B +_v ectors +Ġbrew ing +ĠT k +ĠDownload s +ĠS aved +.Pr ice +Ġcur ved +ĠParen thood +è ¶ +.p nl +plet ely +.D ay +Ġadvertis ers +Ġej ec +Ġpr zed +ë ¯ +! ';Ċ +ĠK ush +ĠT AB +Ġquest s +Ġcoinc idence +umm ies +ĠKash mir +ĠEth ics +_g rowth +Ġakt iv +Ġgroup ing +å¢ ŀ +_tr uth +åIJ ¬ +t odos +is et +Tex Coord +ä tt +ĠZ ur +ro ys +_M AGIC +Ġbrew ery +( State +ĠSM ALL +ĠPl ants +it bart +each er +ĠAd elaide +L u +Ġf ick +und les +_load ed +и е +P oll +rit ic +EL Y +Ġ+ ' +ĠProf ession +Ġst amps +ĠS ew +scroll View +Ġcomm unist +/pro blems +}čĊčĊ čĊčĊ +, o +Ġu dp +Ġob ese +appro ve +ancell ation +_G ame +ĠHas htable +adaptive Styles +Ġpossess es +.match er +function al +M rs +ĉs ave +ĠDb Type +Ġk en +get Context +Ġm ans +( rel +ĠBrother hood +) `Ċ +è§ £ +.In formation +OutOfRange Exception +ĠS ek +C as +Ġblog gers +E ither +(" "" +Ġpin ch +Ġco arse +) p +ĠP ulse +Ġlear nt +Ġdent ist +Ġon change +Ġdirect ives +( actions +ny der +ĠSh ir +T rait +_de p +ĠP ET +ĠRE P +.App Settings +cu ador +iden av +Ġenv i +Ġsl ammed +ĠSh oot +Ġdate Format +.j oda +ve ys +Ġ) .ĊĊ +Ġcare g +ĠPar allel +_ translation +.function s +. obs +Runtime Exception +[] = +over view +ĠSch l +Ġno isy +ĠOn PropertyChanged +S ending +Ġunf amiliar +U pon +ĠPrint s +.t yp +Ġflee ing +ĉm ove +( Un +Ġq r +× ľ +_b eta +Ġsk ies +ĉm e +W ND +Ġstick ers +bl as +Ġinsert s +Ġvers es +ĠD ew +Ġtang ible +Ġhe cho +P OL +Ġte ardown +om nia +IB E +.c over +_str ategy +^ - +set Position +u ale +S igned +Ġif ace +as eline +.set Time +ĠMin eral +ĠFight ing +sk ins +Ġdiscrim in +Ġdans k +ĠPr inceton +ac ist +Ġ( ));Ċ +tr acks +imon ial +ad ecimal +EP ROM +ugg le +.Not ification +$ mail +c antidad +ĠJ ung +Ġseek ers +Ġpl ausible +t ier +еР¶ +Ġr apper +ĠMan a +ĠHttp StatusCode +Ġburn t +los es +ĠF oto +ĠJson Object +Inst agram +Ġsys call +Ġreal ities +ĠMAT LAB +:^ {Ċ +TER M +ĠC bd +ĠPar agraph +Ġtrav és +Ġconstruct ing +Ġsw al +Ġp ige +LL LL +-ex isting +G ets +Ġmelt ed +Ġmitig ate +H en +Ġh m +im as +ĠA o +ĠP erez +ĠD AL +Ġëĭ ¤ +Ġdiv is +Storyboard Segue +ĠMod ify +ĠÃľ ber +_O VERRIDE +.p em +unt os +Ġespa ñ +Ġ{ ? +ĠP AY +_ip v +ĠF ury +__ .__ +el ow +-center ed +check s +_ Reg +-J avadoc +ĉ load +ĠLik ewise +ا Ùħ +UN E +.se m +x cb +ĠC ave +_s leep +Ġsil ently +ĠExt reme +.To Upper +ĉC HECK +Ġc ue +ĠQ ByteArray +Ġcorrupt ed +ĠD é +Ġimp ed +Get Name +Ġinaccur ate +Ġso ber +е е +Ġbar code +-- ){Ċ +ink i +Ġé p +Ġd ri +ĠAL T +>>>> >>>> +ont a +[ L +Ġinter es +ver ting +Ġdi agnostics +p dev +è © +ĠIntegr ated +). ' +_g c +$ text +.g ames +ĠT erra +' Re +.trans fer +_F IFO +get Model +Ġbl and +ĠCole man +Ġpr imes +Ġæ Ī +Ġcross es +n k +G ING +Ġ' ^ +ĠB lob +Ġinter course +ĠBl vd +Ġweigh s +_reg ular +ĠPer th +Ġsepar ating +Ġb illed +.tab Control +Ġpup pet +Ġutil ization +Ġâĸ ł +Ġsucc es +Ġl amps +_pro j +E ric +Ġren ovation +ĠFam ilies +ĠB its +part ials +-M en +s olution +Ġd warf +.IN TEGER +ĠLO CK +. ct +Ġexcer pt +ĠP ix +ĠFirst Name +ANT ED +ĠAd mir +-h elp +P rior +ĠAl ign +.IN STANCE +Line Edit +('/ : +Ġin et +od us +.p kl +ĠK Y +up ert +Ġn erves +_grad ient +} ',' +_un ref +Ġs aturated +ĠConn ected +ĠF N +EX IT +Ġtele port +Ġav ait +Page Route +Ġdivor ced +(l ang +f st +ĠT yr +Ġmess enger +if stream +X S +ĠBank ing +Ġinfect ious +ĠM ons +_LO OP +Ġzur ück +Ġobt ener +/re pos +V el +ac ro +Ġuser Repository +style Type +ĠS RC +VML INUX +rec ursive +/ bar +_ch ip +omin ated +ĠN it +âĢĶ to +ĠBudd h +ом еÑĢ +ĠM AG +ĠC HE +_d en +. raises +_de gree +Ġpump kin +_tem plates +_M EDIA +ĠTim eline +Ġb ots +Object Type +Ġbu ys +.post s +C AL +wait ing +ĠDani els +Ġd abei +ĠS igma +il or +ig el +, W +AD S +( panel +ì² ´ +it ating +.p alette +Ġmos quito +Ġt ego +(parse Int +Ġdes pués +p romise +Ġw ij +types cript +ĠT v +_IDENT IFIER +).ĊĊ Ċ +_fl at +its u +US R +ex perience +-f it +ph inx +_th resh +Ġide ally +ĠFre eman +, DB +_r w +çŃ ī +U b +_stat istics +=" ">< +Ġch ore +Ġy ork +inst alled +Add itionally +Ġp stmt +yl ko +:: Ċ +Fore st +Ġhead set +Ġgall on +ÑĢ ем +Ġwithdraw n +ĠC andidate +Ġmel ting +Ġfree zer +Ġh l +_HE LP +m ime +( /* +Ġth irst +$ return +member of +еР± +ĠHttp ServletRequest +( ob +_ Result +Ġassert ed +Ġfulfill ing +Ġstret ches +par ated +-f unded +Ġå Ľ +ing les +_c a +. condition +ĠDis plays +Ġor ang +ĠC RE +Ġgl Bind +ĠSelect or +/ type +ĠAlex a +ched ules +ĠPen insula +Ġpar ity +ĉ dest +ĠDo ors +čĊ ĉčĊ +_dim ension +Ġa load +.St oredProcedure +(p aren +ĠBur ke +') ]Ċ +- engine +Ġqu ir +ĠHy brid +ĠDo e +Ġout lines +ĠTrend s +_N V +per iments +ĠH in +? ', +ĉ Text +F UL +Ġsm ells +Ġs lick +Ġmis erable +ĠArray Adapter +Ġparam String +H om +_l iterals +us uarios +Ġprompt ing +_l azy +ĠActiv ation +_ oc +We ak +Ġan ecd +ĠU CLA += re +isse ment +ĠEsc orts +Ex cellent +ĠP ause +Ġre positories +T OR +ari ate +_is o +up dates +hal b +udi ante +ë¡ Ŀ +Ġna ive +ĠP eg +ĠL ounge +ARG IN +(b in +On ClickListener +ĠFA ILED +Ġl ite +Ġd zie +ĠL iteral +iv or +fc ntl +Ġe ats +Ġq ed +Un lock +rid ing +und ai += M +AT TER +Configure Await +ici as +ustom ed +Ġsuccess ion +end Time +ĠJ upiter +Ġjud ging +d ration +_d ocs +.m o +Ġeduc ators +ĠV ine +Con d +[ out +q b +\ Validator +Ġmean ings +Ġpresent ly +Ġdiv iding +otten ham +asc ular +Ġtrail ers +ĠC LOSE +ам и +âĢĻ ai +ĠG ain +w or +Ġpl anner +Ġdistrib uting +v at +month s +x label +H F +V iol +.BASE LINE +еÑĤ ÑģÑı +ĠR otate +Ġtx n +: bold +Ġb loss +Forg ery +( embed +Ġjak o +s printf +the ir +Ġexhib its +- static +he cy +get ActiveSheet +.c lients +ãģ į +_h ide +[ word +C b +add Item +ax e +_r adio +al ion +mod ifier +Ġsat uration +Ġden om +_p ixels +m ess +(f l +at if +Ġse cs +Ġpro stitution +Ġgrand children +Ġparad ise +ĠF eld +_B INARY +it ous +๠Ħ +Ġflash ing +-s ided +Ġcontrad iction +/* ĊĊ +y label +ĠT et +Ġadm ire +res o +Ġlet z +ĠSE ARCH +sl ots +ĠRew ards +ĠH og +ĠNS Data +st ash +F all +ĠA mer +Line arLayout +/ photos +Ġfe ather +Ġ| čĊ +Download s +.Start sWith +Ġ// # +ine Transform +Ġaff id +V tbl +ĠRog ue +scri bed +Ġfa uc +ĠMon roe +Ġdecl ares +mod ern +re on +ay be +P ASS +f ers +_MULT I +ĠMath ematics +Ġsud ah +_ATT ACH +Ġnumber With +ĠSol omon +j in +ograf ia +ö l +_d esign +cul ated +ĠL una +ies z +Ġ=> ' +Ġrevel ations +Al ong +( ed +ĠF ilename +Ġy label +Sec ure +Ġbus ca +agn osis +_RE CE +Ġoverl apping +Ext ent +Ġanticip ation +Check s +ĠALS O +or c +iling ual +it ational +Ġadv ancement +ou ro +ĠP redicate +å¾ Ĺ +er ia +ĠPier ce +or io +Ġmer its +Ġpe anut +.P ackage +ĠCon duct +_SENS OR +Ġbo iling +Ġin tra +ĠI GN +ĠF ur +.Ref resh +ĠRe ach +_dec oder +.Ex p +ĠÑĤ ак +p ill +, Q +ĠGr ill +Ġpop ping +.A g +Ġpro yecto +Ġmile age +Ġec ological +] ]);Ċ +ĠÂ Ń +sub plot +ac ad +ĠTry ing +rec ipes +$ criteria +ĠPers ian +-b ound +M ASK +ĠG esture +Ġk k +ĠP VC +Ġprohib ition +Ġcom ando +ĠLO OK +Sh opping +Ġdist ortion +< Boolean +.Get Length +um pt +\ Product +ell ery +Ġfire wall +form atted +.red is +Ġes a +ĠRh ode +S om +.n on +Ġ' ). +Ġget View +ạ n +pr us +Mat thew +Ġs ia +ĠF ors +G PU +ient ras +_IN ST +Ġol arak +Ġimport ing +T CP +/ ");Ċ +e ither +Ġfresh ly +c ascade +(char acter +ĠJe ep +ot ics +_ UTIL +.Xtra Printing +.first Child +ĠEx cell +Ġd vd +Ġt aller +Ġr as +yp ass +Ġassign s +Ġgri ev +-m ore +J D +ĠBurn s +' >čĊ +.D ependency +.Query String +.O wner +Ġexp iry +Th u +( Vec +Ġhazard ous +Ġr pm +AP ON +Ġadd Target +sv ille +p Net +ĠIm g +ĠTIM ER +.An imation +Ġbe k +Ġass ort +Ġle bih +Ġbody Parser +Ġvibr ating +ID L +Ġbutter knife +int ers +Ġpersu ade +ĠLGBT Q +è ĭ +.s oft +Ġbe ams +_s ur +.D ef +Ġl abs +ĉ plt +Ġsk ins +Ġtransf erring +Ġimag inary +_E nd +; background +Ġl aps +_COM MENT +(S DL +ond s +.Rec ord +ĠIm plements +_t icks +() ))ĊĊ +Ġa rose +] ? +ĠM p +ĠI Command +Ġsculpt ure +Ġcontract ed +< HTML +Ġcal end +at y +/ Sub +Ġkv inn +_ IGNORE +ĠSh ane +ML S +Ġstim ulate +Part ition +Ġm un +ó m +eral a +- account +.B inary +c é +Ġse ize +connection s +ĠĊ ĠĠĠĠĠĠĠĠĊ +ĠDi agnostic +V ISIBLE +ĠRun s +Ġimpress ions +s uite +ob le +~ - +ak ukan +< Person +ĠN os +ĠG ui +.wait For +RE SET +Ġpost pon +Dis cover +arr ison +sh aw +b lood +AJ OR +æĽ´ æĸ° +ĠM use +æĶ ¶ +Ġret aining +ot te +Ġmos que +ĠS ne +Ġstandard ized +Ġmain land +_th ree +unge ons +get Doctrine +Ġwh ale +Ġag g +ĠP orsche +now led +lat ent +ĠRel ation +Ġ// ' +Ġshut ting +ĠRem ix +_c ov +Ġs ailing +Ġv owed +Ġp ots +out u +Ġhair y +cast s +Rel oad +Ġre connect +ter a +.child Nodes +ĠR ack +Ġcurrent Index +Ġall en +Ġ çĶ¨æĪ· +ĠC ubs +[ X +_SE Q +_RE MOVE +.get Action +(/ ^ +err ar +Ġ ether +cur ve +Ġsl ap +Ġu om +O thers +Ġen gr +Dis position +Ġst aged +E ye +ĠA ux +auth enticate +Ġ$ ? +ĠAndre as +Ġset w +.A rt +Ġforecast s +Ġa unt +-m iddle +Ġmis d +des k +Ġescort e +ĠCas a +rop ical +Ġexem ple +plan et +(U INT +Ġwh ip +ĠPC B +clide an +=" \ +Ġox ide +Ġsucceed s +der ived +ĠEcon om +_co ordinates +ir as +D raft +Ġvisual ize +B rian +_ASS UME +ĠObject Id +Ġtrain ers +_FOR CE +Ġcon soles +- process +lic her +ĠSim mons +T aking +ĠCl aims +Ġdiffé rent +Activity Result +Ġsn s +éĢī æĭ +ĠCr us +Ġll am +r ab +ĠJo an +AA A +ĉf ilter +ish ops +get ting +à µ +Ġquant o +P ast +ov ich +Ġin justice +ĠF LOAT +Ġal right +\ DB +( GameObject +u ish +(b ot +Ġgall ons +ĠR é +ĠS aid +ĠSTDMETHOD CALLTYPE +ais ing +_process or +ell idos +ter dam +ĠBe am +Text Area +Ġret orno +.M ake +Ġ$ ("< +Ġlock down +Ġremed ies +Ġve el +x ee +do ctype +F il +ĠExp and +Ġemp loys +Ġsession Storage +Ph p +P ublish +Ġret al +f abs +ynam ics +Ġtoss ed +ĠnumberOfRows InSection +x path +\ modules +Ġdis astr +ĠM ULT +.M esh +-st age +Ġs df +it ung +ug es +Ġ?> ">' +kin son +Ġк ол +ogn itive +_ li +Ġim minent +Ġaff inity +.sign al +Ġnot ch +ĠSteel ers +max length +K K +ĠEug ene +_P WM +ro i +Ġâ Ĺı +ĠH amburg +.M ust +Ġax e +en ef +Ġamb itions +ĠSpec ies +ĠSt ress +Ġa while +Ġб Ñĥд +Ġwith stand +ĠDec oder +_in ventory +Ġ{ ččĊ +Ġt gt +Ġrail road +W ASHINGTON +Ġnegot iated +N ST +- phone +, U +Ġexerc ising +á» ¥ +_P IXEL +av ors +iter ated +Ġv ampire +ad al +In grese +Ġun g +ject ive +.c ells +Ġn ano +Ġmark down +_R ULE +(event s +Ġl uggage +MESS AGE +ig keit +$ count +Attribute Name +IG INAL +_E nt +ĠB F +ĠCOM MENT +_in i +ĠEurope ans +ĠB elle +åij ½ +) [' +åº Ķ +ĠUse ful +.re ference +() ", +_ grade +ĠK aw +Ġsent encing +Ġsocial ism +mon ster +_L AYER +Ġdee pest +w k +ĠNo ise +### ĊĊ +Ġpr éc +ot le +ÑĤ е +a uf +ib al +Ġcon quer +> Email +Ġamb ulance +O AD +Ġ(" % +ĠF I +.f ixture +Ġter se +ĠĠĠĠ ĉĉĉĉ +Ġsanct uary +ug i +ĠCom parator +Definition s +Ġast hma +Ġl act +Ġhard wood +.c lock +Ġattract ing +ĠM our +(d istance +ic its +Ġbon ne +ĠAC CESS +.Deserialize Object +ĠTyp ed +Ġje u +Ġapp Id +ĠCl ara +ĠH F +ĠRe ich +ipp les +//---------------------------------------------------------------- ---------------- +_del ivery +erial ization +Ġplaint iffs +Sc ient +sh opping +ĠD ummy +ĠW ald +Group Name +Ġins cription +el og +:::: :::: +_ ld +Back Pressed +.R aw +ĠOn Trigger +Ġmuse ums +ĠBe en +ĠAdvent ures +Ġsl ate +Ġlet t +Ġsu nd +ĠG in +ĠMechan ical +.s hip +App Component +Ġdest ined +Ġdw elling +Prof iler +Pre pare +ze ich +Ġsil icon +(h as +Ġ# % +VID EO +Ġcollabor ate +L in +Ġsc opes +( className +(s d +and in +.h am +Service Impl +-des cribed +Ġiron y +st ial +ĠHu awei +(re po +Ġunexpected ly +ĠK ai +.inst all +\x f +Ġexhib ited +_T CP +ĠO x +_CH O +Ġprostitu erte +Ġv ä +Ġsit o +Ġconstitu ents +ĠContin ued +ĠS AVE +r ss +/ message +ub es +Ġmisd emean +Ġtax ation +Ġstory line +h air +ĠFind s +S IG +ver ification +~ = +.h p +Iter able +Ñĭ е +ator i +Ġc tr +R x +_ );ĊĊ +d ag +.p in +Ġp seud +Ġinv o +ÑģÑĤ ÑĢ +_p ix +为 空 +Ġsw orn +âĢĶ or +_reg istry +Ġdis asters +ĠRO I +ĠâĢ ķ +akt u +fore st +be iten +âĢĶ I +ue va +eg t +Ġsp ikes +URE S +ĠRecomm ended +Ġexplo ited +ĠFreder ick +_COMP LETE +ĠDr ugs +!!!! !!!! +ĠR iv +ST OP +RO OM +ĠP ASSWORD +C ookies +.E l +á» Ń +ĠB ert +Ġhash ed +ic ester +Ġdecor ator +Ġquery String +: ;Ċ +Ġ" [" +oto pe +-A meric +ĠMatthew s +UR AL +âĢľ , +Sum mer +f os +_CONT AINER +_A CK +Ġfil tr +_dis p +_ Re +Ġfac ile +а ÑĪ +Ġìķ Ĭ +Ġe ben +Ġspr ink +ĠQ uint +> V +Ġhistor ians +our met +ĠMonitor ing +led ger +c ott +Ġw are +GG LE +c ars +ĠM EDIATEK +Ġvol upt +_ View +HE L +(c opy +(st ats +Ġchrom osome +ĠCurt is +- conf +( asset +Ġhv or +File System +< >();čĊ +oc oder +ĠC annon +) x +ĠSm ooth +ĠS AS +_ ce +ĉ prev +_m ovie +E c +_w all +< Button +ĠF AST +Ġon View +ul an +ĠS UPPORT +Ġgesch ichten +ĠS ons +Im m +$ IFn +Ġfair ness +Ġd pi +ats u +J osh +Equal ity +Ġ} ()Ċ +_ less +ĠR atio +ĠC ats +ĠS tern +Mon ster +Ġmer cury +ü hr +Ġplus ieurs +.des erialize +sc opy +.F alse +) animated +ĠExp erts +Ġ"") {Ċ +.W hen +see also +.un pack +LE M +.select All +Ġperception s +ud ing +ir ling +ĠPrint ing +gram s +ĠFile Stream +erv ille +il og +ic mp +_C ount +Ġlivest ock +- ca +doc uments +Ġpo les +ĉw ant +Ġflu ores +Ġstand point +ĠH uge +Ġradi ans +ĠUIB ar +EDI UM +ĠHistor ic +_h older +ĠMar ines +Ġt ä +.L ight +quir er +ason ry +div ider +ĠFl utter +_f b +restrict ed +ĠEvery body +N ão +Ġkn ot +ĠT witch +Ġhall way +(C ollider +Input Element +? )Ċ +/ off +/ ) +play ed +[ OF +Ġbat ting +_d l +Ġcom edian +Ġé v +ĠD EM +ĠEd en +: white +' ', +Con struction +acer b +Ġtask ed +.man age +Rel ationship +Ġph on +n z +_B GR +Validate AntiForgeryToken +_ air +âĢľ When +Ġgl fw +ĠCon versation +_T OTAL +, Z +Ġg raz +Ġiter able +ĠP ASS +Ġadvert ise +Ġmö glich +/ train +ĠVolk swagen +Ġcreep y +Ġ" )čĊ +QU ENCE +Ġalt ar +Ġed its +comp iled +aw ning +ĠD ungeon +Ġo sg +Navigation Bar +Ġtrend ing +ĠE co +ogg les +cd ot +| - +S ie +ec ret +ĠN egative +ĠL ing +ĠD IM +ĠC WE +ĠCar rier +Ġcar tridge +_us b += os +ĠJack ie +Ġo tras +Ġcommod ities +ĠP resentation +)&& ( +ĠMar tha +ĠCath olics +ĠM ond +об Ñĭ +_ absolute +Ġash amed +pons ors +t al +Ġsad ness +Ġpu ò +F ade +-pre view +ĠRequest s +ĠCal vin +h orn +Reuse Identifier +(pro vider +/app s +ime o +ĉ Class +S amsung +ĠW ORLD +Ġc innamon +dot env +ĠI User +ĠDE V +_C har +.ib atis +et i +/ me +s st +.s ym +ĠRug by +-m aster +aj ar +ĠY EAR +Ġo dp +ĠR oles +Ġbip artisan +ail le +Ġblock er +Ġgre ens +.SE CONDS +Ġbelie vers +ĠL ikes +F LOAT +Ġm ak +Ġg cc +âķIJ âķIJ +(" ~/ +SCRIPT OR +Ġton nes +ĠS ang +Ġtrans pose +enn ai +P red +Ġsoll te +.github usercontent +( print +ĠH ole +çľ ĭ +ad get +Ġprompt s +Ġgen etically +ĠH od +Ġvert ically +_control s +ÑģÑĤ ан +") {čĊ +$ title +Ġ} ),ĊĊ +Ġstate wide +ĠCor respond +ĠAt tr +it ant +Element Type +Ġout ward +Ġfam ilia +( article +Ġbl at +Âł Ċ +Ġgl Get +ĠRe ceiver +Ġ% - +ad am +W inner +Ġtail or +_p wd +ert en +St an +ĉ all +al ive +strt otime +� s +s essions +$ conn +ass ist +Ġchat ting +ĠM ant +Ġ% @ +Ġ"" );ĊĊ +Ġd gv +Ġíķ ¨ +.re peat +_M essage +Ġadvis ers +/ path +Ġk es +) } .ĊĊ +ogen esis +ĠOPTION S +upt ools +Ġmilit ant +Ġex ited +ig ar +ĠCOM M +ĠDis posable +ay cast +Ġrow span +Ġsyn thes +Ġsond ern +ĠĊ +ĠJ acket +R ATION +.getSelected Item +- init +ĠReg isters +_se p +ĠTool kit +.d ict +Ġx label +\ Table +t oc +_com bo +ĠComp act +Ġr ugged +à¥ĩ ठ+-man agement +')}} ">Ċ +ĠSt amp +ı l +ro x +Ġlandsc apes +_NOT E +mon ary +c ab +Ġmo et +x af +rc ode +- cli +_g ate +[ event +SP ORT +g ia +ĠS UPER +/ Login +_sh utdown +int errupt +Ġpret ending +Ġfr inge +ĠRed s +ĠC UDA +ĠUN IX +v it +Ġbr ig +dr v +ĠConn ector +There fore +Ġl ia +D etection +_ actor +Ġtemp file +Ġecc entric +- role +Ġpad x +d ent +West ern +Ġê ·¸ +ĠApplication Record +Ġcampaign ing +_run ner +ĠC ivic +ale igh +Ġdire kt +.s ul +ĠĠ ĉĉĉ +ant en +Ġiss uer +Ġassert ions +( orig +AT IO +Ġlean ed +ä s +.D TO +expl ode +.O bservable +Ġstagger ing +Ġkidn apped +Ġprogram mers +ĠInn ov +.param eter +Ġdom ination +Ġske ptic +Ġæĺ ¯ +Ġavoid s +.Ver ify +ub by +ĠAS N +Ġformat o +ĠBeat les +_b rand +Ġin set +y outu +Ġto c +-f inal +Show ing +ĠD oub +ĠM esa +Ad j +_m edium +Cre ates +(end point +ĉ UP +bb ie +Ġst alk +.datab ind +.S can +ag ents +$ , +ind ividual ++ )/ +ĉv m +(not ification +Ġin ex +ĠClass ification +ren o +Ġo lig +-r ated +Ġform ulation +', { +Ġa cept +_un pack +_C A +.P ow +ĉ im +Ġal uminium +AN O +Ġx n +Ġcó mo +ĠIng redient +Ġseiz ures +åħ ± +ific ador +Ġsigu iente +ĠIn fragistics +Ġduplic ated +ĠDe e +Ġn ø +ĠAC CEPT +(c rate +иÑĤ елÑĮ +- less +Ġinf inity +An alyzer +-D ay +rit t +(c in +ĠG y +Ġmulti plied +uch i +ĠBald win +/ ip +Ġshort cuts +.A DD +Ġvig or +_in struction +( ; +_ eta +è¿ ŀ +utor ials +Ġboost ing +b v +Ġacknowled ges +List ening +FA Q +; b +(( - +Ġarchitect s +Ġz we +Ġpul s +Ġget Count +ver bs +ãĢ ľ +(C ollection +k re +Ġjuris dictions +_b ridge +ĠCr ack +ĠDiff iculty +K O +Res ervation +_re quires +T our +ãģĹãģ Ł +.set Current +Ġk y +ĠAlb any +Ġè § +ll er +agn a +work ers +.bl ank +ĠPr ayer +M IC +Ġresil ience +Te X +ĠL anguages +st udy +ĉc urr +Ġenzym es +Sl ug +ĠíĮ Į +str al +Ġtum ors +Ġseg unda +=' { +in struction +ĠL isp +/ info +Ġ" {$ +,: ), +Ġg v +( ErrorMessage +Ġ' = +}- ${ +.Doc uments +" Well +Ġreminis cent +Ġg az +iro pr +eh r +Ġsup pressed +ers h +.scroll To +Ġcad ena +Ġgame State +ÃŃ m +( conv +ĠTom orrow +ĠC CT +M ongo +ul g +.C amera +.hand lers +m ph +Ġst k +Ġgen etics +AC ING +Tr ivia +ĠB am +(m arker +.St retch +ĠSun ni +ĠBet ty +.t olist +un likely +.Rect angle +ob solete +IL ON +inner Text +emb ourg +a N +ĠV ehicles +un lock +: utf +n ob +ĠSee ing +ĠNE VER +Ġt ls +Ġfil les +Ġbenef ited +ĠCl int +*/ ), +.f old +Ġpos ible +A DED +th ouse +.D AL +ĠO dd +ro kes +ĠSun ny +ĠPartial Eq +_B uffer +ĠLe vi +long rightarrow +eld on +g ages +_w arn +.Create Table +ĠD ip +_ questions +.log ic +Ġ# " +={() => +Ġt ep +Ġju icy +ì Ĥ¬ +en ko +ia lect +Ù ī +Ġon board +Ġæ ı +ĉ rt +_ UTF +ĠQ Action +âĢ ŀ +( Component +(a udio +.h it +g te +Ġprogram med +state Params +Ġpoly ester +f ires +by ss +] =( +_ quality +Of Day +ĠFair y +Ġy elled +op l +(user Name +ĠD ifference +Ġevalu ations +iff any +Ġcycl ists +Ġc idade +Ġtext book +Ġprof iling +__ ), +de a +. activate +Ġindic ations +Ð ķ +Touch UpInside +Ġinval uable +ĠM ASK +Ġcont end +F req +Ġrecru its +(int erval +ĠUser Profile +Ġ'./ ../ +ed u +_C allback +Ġanal ogy +ĠTro phy +app hire +V ideos +ĠCh er +ĠH av +âĢ¦ " +. validator +g fx +ĠU Object +class names +tri angle +ĠEnc oder +.s py +Ġpred ators += status +-s afe +: ",Ċ +ĠIn cluding +Ġ{} ;čĊ +* cos +Ġend ured +.sul ake +Ġnurs ery +Ġfrag rance +Ġre building +Ġn th +ĠFr aser +.set Date +ĠV ince +_RE ST +Ġvent ilation +æµ · +cri bes +.as m +lp Vtbl +ĠA be +uis ine +, array +ĉ className +err als +Ġ' ĊĊ +Check out +Ġsol icit +A ux +_c apture +Ġrib s +rag on +vi ol +top ics +Function Flags +ĠM arty +b ike +ĠT ucker +(k ernel +ĠO ps +Close Operation +/d emo +ild a +ĠlÃŃ nea +APP ING +Ġsu ites +.visit VarInsn +ur us +ĠMin ute +(m anager +Ġbutter fly +Ġap are +Ġw olves +J WT +ĠSal on +ĉd elay +-es lint +is ations +.r pc +)| ( +ĠSnap chat +/m m +M N +cer ies +.text Alignment +ĠFrank furt +Ġad o +(new Value +( access +( Expression +ĠSign In +ĠHait i +_t p +.set Parameter +Min ute +Ġmanual s +ric anes +ĠP TR +ĠOut er +Ġget line +oc ations +_C D +ĠLy on +/g ui +_l ive +id an +.ge om +Ġborder Bottom +im uth +_check point +Ġme u +ĠIr ving +Ġpeu vent +(M AX +ĠAR CH +Ġp ov +.source forge +Ġjam ais +Ġar k +ĠBaghd ad +ĠC LEAR +Menu Bar +Ġtro is +CHED ULE +Ġ# čĊ +(C all +$ order +(M aterial +Ġencontr ado +$ list +ĠMETHOD S +.begin Transaction +_M AG +Style Sheet +Ġmaj ors +Ġindef initely +clean up +Ġhom eland +(d to +D ates +P resentation +ĠD K +={` / +ĉ Key +( Block +_check box +ne eds +Ġon Complete +ric o +Ġgle ich +Ġx m +O OD +B etter +ĠSQL ITE +. Book +x ad +ĠG one +ĉd p +Ġdev otion +Ġst m +Ġobs ess +ĠBack end +Qu eries +I k +// **************************************************************** +Ġdivid ends +.parent Element +} ")ĊĊ +ĠMaterial PageRoute +: num +Ġexp lic +ĠO L +le ast +O ops +iment os +Ġins urers +Ġhero ic +ĉf ields +.img ur +.btn Cancel +ĠDetect ive +(s m +ĠMutable LiveData +.l ab +(( [ +Ġha irst +ĠTrans actions +å¼Ģ å§ĭ +Ġstd Class +uent o +G IS +_c od +Instruction s +C alls +Pointer Type +ĠR w +Ġassort ment +ĠD IG ++ r +_C ERT +Ġinst ability +Ġv ib +on as +Ġro ku +ap ellido +Ġan gl +prene ur +Ġfluid s +ise ase +Ġde ed +qu ist +_CONST ANT +Ġequ ilibrium +_de legate +ĠQuant um +re i +Cap abilities +rect angle +? >< +al ien +ĠJ ug +D NA +T ickets +Occ urs +ĠHaw k +.setHorizontal Group +\ Collection +ff iti +Ġre arr +.setVertical Group +Ġc avity +Ġadult e +Fac ade +- wh +ĠL OL +Ø ° +Ġgrand parents +Sw ift +ĉw x +æīĢ æľī +if en +ff set +B eyond +// }ĊĊ +Ġw ager +Ġb ury +Ġcomm ence +reg istro +sc ient +ĠPer cent +Ġд олж +( identifier +.set Model +Ġs eldom +nt on +Ġappl iance +am us +rys ler +Ġpant ies +engu ins +Ġmim ic +Ġon Changed +Ġal coholic +.reload Data +Ch arge +ĠF ax +Ġj ScrollPane +Emp resa +Ġsh attered +x ba +Font s +? s +Ġpost season +ret ain +_r ates +Ġrequest Code +.t odo +´ s +CH K +ĠKeep ing +enge ance +Ġvs code +IPP ING +Default CloseOperation +_ raise +ĠO culus +ogram s +ra j +pc i +Ġcorros ion +.handle Submit +Access ible +ĠP iano +l ittle +AC L +Äĩ e +.un wrap +ĠCon vers +ĠLe ben +ione er +ĠMer chant +ĠJ orge +Ġembr acing +Ġvent a +á st +Ġvi ene +< QString +Ġexplos ions +Ġdistur bed +." < +m emo +ĠAb original +Ġcomple to +Tex Parameter +Ġuom ini +( agent +Ñĥ ÑĢ +ĠWh olesale +/ am +ĠBook mark +dr agon +Ġglo ve +Ġ" "));Ċ +iv ariate +now rap +In Children +.B r +Ġcon exion +Ġback bone +Ġe clipse +Ġpersec ution +': ĊĊ +/ link +ĠP ero +and as +ĠT ek +. "); +-an alysis +Ġer ad +Mar shal +Ġanch ors +og er +Ġconver gence +st icky +Ġnave g +int ern +_DE SCRIPTOR +ĠConsult ant +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ĠA uch +Ġer re +ÅĽ li +ĠHor izon +col a +Install ation +hot mail +C NN +.C ollectors +ch s +(tr ace +ĠEnc rypt +Ġ---- -- +ĠBase Controller +Ġag ua +Ġre active +id l +Ġclass Names +ĉ Session +ĠDod gers +H ad +_l v +Is Valid +ĠHEL P +ut to +ĠVer ification +Ġget env +_p a +.b mp +: f +ĠLou ise +(' ; +/ socket +Gr anted +.c alendar +( IP +ĠP X +.R oom +Ġprogram m +ens i +Ġtablesp oons +Ġle ve +Ġmo str +.t ipo +/ an +(d i +Ġb iod +Ġdb Context +ĠJS X +ĉ results +. END +ht e +l ify +P recision +èĬ Ĥ +ARS ER +)did ReceiveMemoryWarning +at tempt +IS P +& a +_P OP +ĠT ac +Ġprepared Statement +Ġзап иÑģ +Ġow ing +, start +Ġreview er +Ġr st +Ġprop Types +Ġrock y +_lo cale +ĠStrateg ies +ĠWe ber +.C ascade +_equal To +Ġcos as +ĠDe letes +ĠMax im +Ġsh rimp +re trieve +.In clude +IG IN +ĠO E +] );čĊčĊ +.en umer +Ġco ef +_N ull +R a +ty ard +ĠSh awn +keep ers +Ġq q +_s b +om ens +ĠExec utes +# " +TT Y +ĠValue Type +); */Ċ +ĠAbs olutely +ĠT ottenham +/ art +Ġbless ings +Ġswift ly +b uster +Ġa vid +COM M +, temp +Ġ} ?>Ċ +-g rowing +Ġdeep copy +A ck +egg ies +Ġ__ (" +Ġno ir +terror ism +Ġanth em +ag ency +_PACK AGE +ĠC losure +.reg istry +Ġmamm als +< L +U ICollectionView +ĠLED s +Ġvol ley +( Buffer +_N ATIVE +lib c +impl ode +Scroll Bar +ĠMar ion +.Con tracts +_A t +ĠWe instein +compare To +ĠH ose +en ity +.create Query +_r outer +Ġstim uli +Ġ++ ) +ĠCh amp +ĠBay ern +ass a +.v a +Ġdistrib utors +Ġfile private +Ġdepart ed +cc cc +@ click +ĠL unch +> L +Ġbl uetooth +.De ep +- standing +ác il +Ġro oft +ĠPath s +_iter ations +Invalid ArgumentException +.s pi +ĠUIAlert Action +uy e +sign in +.p riority +ĠEss ays +=' {$ +Ġè¿ ĶåĽŀ +_s igned +.p ersist +Ġred esign +To Lower +ĠNew man += start +ĠIsrael is +asis wa +Spe ech +Ġnum eros +hand lers +ĠW ong +Ġм еÑĤод +We ights +ĠGu jar +te il +ĠNon etheless +_E FFECT +Ġv ect +ĠO sc +Ġco ats +ĠW heat +Ġge ek +ĠPRO PERTY +w orm +_const ants +ĠB oulder +ĠP arm +co le +Ġdefault Center +ĠRou ge +: A +xc f +ĠVen ice +med ian +Ġred emption +F resh +Ġcos m +Ġfig ur +Ġref urb +CO PE +.c d +Ġch ords +ĠS gt +Å į +VP N +ĠS END +ain en +_account s +Ġtent h +Ġdiss olved +< App +ĠCover age +use State +é ro +.. < +Ġì £¼ +Ġdream ing +ĠFore cast +.C ursors +Ġvis as +/ script +_start ed +Ġga str +(P RO +]; // +.T ile +* sin +( Adapter +ĠSand ra +_S IG +ard ash +ĠO val +Ġdescri pcion +(s l +ĠDes criptor +Ġ` $ +/f ree +ĠKey words +Ġt udo +ion ale +(f ound +.x yz +ĠGeneration Type +_DISABLE D +( area +Ġel ites +Ġh ombre +(m essages +ĠR ac +Ġext ingu +ĠEst a +op o +. vel +mouse out +Ġconv olution +ĠHand ling +Ġceil ings +T ek +ĠAre as +.writer ow +< View +ĠCorn ell +_B IN +.in valid +'' 'čĊ +ie ż +_P osition +Ġk idding +PC ODE +Ġwatch er +lo x +Ġâ Ĺ +D ave +_all ow +Ġbis exual +Ġun ordered +ĠSch we +_se gments +Ġt earing +IN LINE +Ġund es +.g oods +.c am +ĠL W +ĉ where +Cal culator +-th reat +- alert +ĠSuz uki +ĠIP A +ĠAtt achment +AC CESS +(d type +O pp +_s ymbols +Ġdans ke +l age +or get +res olution +е Ñĩ +ĠQ Color +ĠBar rett +аÑĨи Ñı += \' +ĠNav Controller +/ ref +(c ountry +_H DR +Ġterse but +pet ition +Ġsu f +cred its +๠Į +x m +ĠDav ies +.re ddit +Ġw oven +ĠO bl +ĠK M +ĠConsider ing +ens ored +.per iod +Ġd dl +$ wp +Ġextrem ist +; \Ċ +Ġk im +al ers +Ġspan ning +Ġco herent +Ġconse gu +.text Label +.g eneral +_d ashboard +л ение +k ick +_P ID +ĠExt ensions +reg exp +ĠCl ause +_m ov +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +ĠR eward +ĠLEG O +A k +=-=- =-=- +ĉ parser +Ġon ze +éĢ Ģ +âĢĿ ãĢĤ +_b all +(r hs +Ġch orus +< count +as urable +Ġwirk lich +ĠEr in +ĠMS NBC +Ġet ter +ĠC ron +_F LOW +Ġ, čĊ +Ġcal idad +ĠFile Writer +ĉ stmt +( Byte +_p at +Ġte lescope +Ġgre ed +ĠT ort +(w rite +\ application +ĉRT LR +ĠConfiguration Manager +Un ix +End Time +In cludes +ĠHar vest +en berg +ĠAustral ians +Ġë ĵ +Ġr n +Ġreput able +Ġbl ending +UL ATION +ĠBrend an +d ad +Ġm ø +ĠW oo +_d c +U ne +Ġr ue +with in +ang ep +Ġp ouch +\" ", +ĠS ic +âĢĿ ), +aly ze +ĠG ef +c overs +Ġd bo +replace All +ĉ Logger +Try ing +[ state +-p iece +éĸ ĵ +beh avior +all ows +l rt +_p ython +ert ura +-c ountry +ĠT G +.UI Manager +b ens +ale x +ĠBre itbart +b ac +Ġpredict s +Ġg ab +Ġcard inal +.Time Unit +ĠVis itor +ĠM ing +Ġliv re +Ġparent Id +port un +Ġdimension al +ĠV est +en ic +à ³ +Ġ Ùĩ +ĠBL UE +Ġitem Count +Ġfe athers +ĉp stmt +ĠPol ar +{ // +und i +Ñĥ ж +z ar +Error Response +ì ĥģ +Rep resentation +* _ ++ ] +pre pend +Ġ' > +Ġlegitim acy +Ġo o +S linky +Ġnation als +. words +; p +tr ap +oman ip +Ġc ues +Ġgradu ating +Ġsem aphore +"] );ĊĊ +ace y +RE ET +Gr ab +ĠFel ix +( Id +_ne ighbors +Ġmeaning less +(d el +Ġj eder +ĠContent Values +.abs olute +/ cl +Ġx b +dat um +Ġtort ured +Ġrub bing +S cores +ĠðŁĺ ī +Ġav ons +Ġam sterdam +E OS +H al +Ġtrust worthy +# = +.EX TRA +Ġman o +is icing +-s upport +ĉc ursor +ĠSp o +aim assage +M ission +[] {" +Ġprint ers +G REEN +Ġt eg +Ġabdom inal +! ĊĊĊĊĊĊ +.Sh ort +аз в +ĠGift s +} ") +(b inding +x ce +âĢ ij +inf os +Form Data +Ġd art +Ġele ms +(in v +Y L +t in +GEN ER +á» ¯ +ĠT aken +uck le +: e +Ġspect ral +.b aidu +/ ');Ċ +Ġgre edy +es ion +,,,, ,,,, +Ġ/> ,Ċ +Internal ServerError +NSNotification Center +ĠA i +Ġsp it +Ġaug mented +Ġstandard UserDefaults +FIN ITY +R ace +: C +ĠRE CORD +ĠHigh light +Ġ' ` +Ġdef icits +Ġne i +Ġresearch ed +T a +Ġc opp +.Get HashCode +): čĊčĊ +On Click +ĠWell ington +Ġrev ival +æ¯ Ķ +éĹ ® +ĠN SS +Ġfor n +Ġint é +ĠKu wait +_fl ip +_ bo +_ \ +Ġocc urrences +ĠScient ists +S RC +og ens +igr ant +RE MOTE +ĠS ID +. opts +u ve +() ])Ċ +Ġlibert arian +ĠGl ide +les en +Ġform e +ow ania +Ġannoy ed +Def s +ĠExec utor +Ġcast s +.set Checked +ĠSh aring +.Serialize Object +Ġselect ors +_ OTHER +ë¯ ¸ +(s uper +( OS +_VER IFY +id unt +< header +Ġ/> ';Ċ +Ġvidé o +ĠNeg ro +ĠL ords +ĠT ours +Ġsoft ly +.re ceive +ĠE RC +Ġdata Set +Bad ge +ĉ Event +Ġper l +Ġ{} \ +(s entence +Or Update +Ġdim inish +P IN +(d raw +.To DateTime +.Equal To +(p in +-p encil +lu ent +ĠCall er +Ġplay ful +- '+ +x ca +sw ick +){ }Ċ +}: ${ +ĠM eth +.get Cell +.b reak +Ġy max +=' Ċ +ĠH iro +( TRUE +as urer +Ġcu er +U ber +. Operation +Ġol an +Ġthr illing +< Response +ĠF emin +Ġtravers al +Ġp oc +Ġset Status +decl ar +std afx +Ġaddict ive +ĠB tn +Ġexplos ives +ĠCook ing +ĠPl aint +Ġaccum ulator +ĠApp ointment +, password +ĠF AR +lu et +Further more +decl spec +_Static s +.D ictionary +"> '. +ĉ valid +" ", +In strument +> J +Ġno str +ĠR ift +_P ort +Ġvec es +[ [' +Ġrall ies +- series +Ġv v +. uc +Ġr tn +State Changed +( ins +ĠCl a +------------ Ċ +c us +ĠRel oad +//---------------------------------------------------------------- -------------------------------- +.se conds +_dest ination +Ġscrew ed +> c +Th ickness +Design er +Ġgr ids +n Äħ +( cookie +T rip +-M obile +Ġv oll +Ġgen ital +Ġconf isc +ĠConfeder ate +Ġweb View +Ġm ise +Ġcl er +(se lection +$ date +Ġshar pen +rag en +And Update +Ġrem ix +Ġh tons +R W +M PI +Ġretrie val +Ġric hest +.Dec ode +:init Components +ĠT Value +S aint +@ include +ĠPER SON +.se p +ĠLD AP +g ba +Ġgro ÃŁe +Ġreli ably +ĠD FS +.getItem Id +Ġprés ent +.get Token +Ġch inese +ĠMe al +Y OU +"> >ĊĊ +b ower +Ġsw apped +/ install +Ġs inks +etr ize +Ġdecl ines +ĉm ysql +ĠC String +ĠMotion Event +.L anguage +R oad +ÑĤ еÑĢ +asc imento +')) -> +. about +( editor +ĠR atings +in come +Å¡ e +.de queueReusableCell +ĠAust rian +Ġs ulla +ĠTrib unal +ĠDid n +ов аÑĢ +Ġins pections +B oss +Ġcock tails +Ġapolog ized +_sub plot +op al ++ =( +Ġreson ance +ib u +Ġë ¦¬ +rom a +res erve +pl s +ĠT ah +ax ies +OP LE +ĠDar ren +ĠZ ombie +_M ap +Ġ] )ĊĊ +ĠQ i +ĠS ail +Ġrestrict ive +Ġeros ion +- par +WH ITE +Ġold u +Ġap erture +Ġbit coins +text o +ĠCom cast +Ġtime less +en kins +Ġfeed er +/ tmp +res den ++' _ +.D estroy +Ġç ok +ĠD OCUMENT +.l ng +.tag Name +Ġk ullan +eg rate +Ġ(* . +ç¼ĸ è¾ij +Ġhand shake +s oc +_ geometry +ĠDam ascus +Min or +ĠK afka +ìĹ ¬ +Fl orida +_com pute +.ex pr +Ġpar alle +ĠD iaz +c ir +[ target +Ġj oking +Ġgl or +(set q +_hand lers +H ang +Ġf err +rim inal +ĉĠĠĠĠ ĉĉ +ent ies +def ines +-t ax +json p +ĠU PS +met ro +__ ;Ċ +ĠUg anda +])) :Ċ +_t d +x ae +l w +. OS +ĠLog ged +ac id +ĠMay o +as pect +Ġvag inal +Ġinitial izing +Ġster oids +f iction +G RE +g end +Ġli abilities +ĠL ets +M ech +( nc +( change +Ġconnect ors +: k +Ġt ast +! ");ĊĊ +th ings +ro phy +luet ooth +ĠSign Up +. ctrl +Ġthere in +ord a +. escape +ig ator +Ġpet rol +Ġspec imen +Ġdeb uted +- Pro +Ġcr ises +.add View +ëı Ļ +-d oor +Ġmon et +Ġmill is +Ġv ier +Internal Enumerator +Ġadmin s +ĠL air +z in +get Query +umb les +L IMIT +ĠV ig +_s ong +< Character +:: . +_h om +_b p +ĠSup ervisor +sub mission +ab ile +Ġno i +Or Create +Ġpe el +Ġon Start +Ġsent iments +veh icles +Ġclass rooms +Ġs zer +Ġb ending +Ġlong evity +Ġa cl +ĠAle ppo +ĠU M +ĠR icht +Ġmultip rocessing +DOM AIN +"," + +_Y EAR +Ġsc rape +Ġsol itary +Ġ"] ";Ċ +/ errors +ìŀ ¬ +ľ ëł¥ +b etter +ĉ number +ĠL F +ĠAc ross +Pub Med +\" " +ĠExcell ence +Ġus ando +ĠU IP +Activity Indicator +_V OID +Ġbre eds +ï½ ¥ +uest as +ĠTre asure +ustral ian +(f ace +ĠT ennis +ĉ Int +ĠHans en +ç µ +: I +Ġâľ Ķ +GR AY +O USE +Ġhe pat +ł í +A IR +ó ż +Ġque ued +vinc ia +ĠChrom ium +Ġcompet ence +ung al +ill i +Ġget By +ĠF inder +Ġincap able +Ġs add +Ġc ites +ĠChurch ill +S dk +More over +As pNet +( Float +$ password +ĠConn or +-s ession +_d m +* )) +Ġde utsch +ĠN X +Ġper ks +_S ORT +_TO OL +_V ISIBLE +.as p +æĪ ĸ +ĠBre ath +D etect +ĠD uel +.c mb +[ it +.Set Bool +Ġnarc iss +Ġab ide +Ġej emplo +ĠâĦ ķ +Ġm ornings +Ġcomput es +.s sl +j t +Ġmuch os +_S S +[ end +Ġbas in +Ġalgun os +ĠCroat ia +lin ewidth +(t ags +(h idden +ÃŃc io +Ġap ar +ĠÐ ¶ +ä¸ İ +. food +ĠR ural +Ġbread th +å½ ± +(s ess ++ ") +ĠP aste +Ġserv idor +ĠBit Set +ĠTr an +la us +v ette +ey es +ĠCL ICK +ĠV III +ĠTurn s +ĠLe Bron +ĠM uj +ĠD eg +ĠAdult s +_s uite +process able +ĠPH Y +g hest +.F ail +ĠSl ack +ce j +\ Carbon +Ġsuper star +Ġhold ings +( forms +Ġ'# ' +M ultip +("[ % +-s olid +/ url +-t ier +[ length +ĠStream Writer +ĠMarket place +get text +_T ICK +ĠFor ge +Ġblack jack +ĠDO ES +ĠM atters +w aves +Ġwhisper ed +Ġl ush +ìĺ ¤ +d igital +Ġwr ink +ĠH ogan +Ġrust ic +.Apply Resources +ĠHard y +os omes +A UT +.ST ATE +Ġnarr atives +ĉ store +b ib +ĉ Scanner +ĠC ody +\ Repositories +Ġre union +and um +âĢĻ h +Ġsn iff +NS Bundle +Ġcompreh end +_US AGE +_ occ +URRE NCY +J NI +Ġspecial izing +Ġvis ions +Ġdol ore +Ġv á +ĠChe vy +ĠSt yled +imp act +all en +Ġk art +ĠTable t +st uff +re esome +аÑĤ оÑĢ +//---------------------------------------------------------------- -----------Ċ +_Ad min +Ġcell phone +Ġaut oplay +Ġcamb io +Ġmar itime +_BO OT +- quarter +Ġlat ina +ĠAJ AX +e quiv +ĠFront ier +ĠX Y +} ]Ċ +ĠR ough +.pro to +Ġcorrect ness +Ġfac il +ĠRe ached +ãģĿ ãģ® +V IS +.p s +Ġstr ncpy +Ġdiff usion +.start Activity +�� � +Ġaccom p +AMES PACE +imon ials +ĠBl ast +aby rin +Ġd ome +Ġextr av +Ġy en +Ġcul inary +P RI +ĠComm unities +n id +_oper ations +.h s +ĠMil ton +Ġno ises +Autoresizing Mask +(c id +}ĊĊ ĊĊĊĊ +] },Ċ +ĠD etection +tab la +Ġlib erties +_D YNAMIC +w get +ĠT ür +ĠP ascal +Trans parent +Delay ed +] () +ĠHer bert +< ActionResult +ch allenge +Ġmush room +.insert Before +ĠR in +Ġhum our +Ġf ø +api Key +alloc ated +Ġconf ession +. ",čĊ +ĉassert That +ĠS ORT +ĠL ORD +Ġexport er +.set Level +p okemon +ash tra +Ġf é +ur ator +(M SG +Ġt up +ĠH ull +Ġyield ed +.Sub ject +\ Route +! ? +ĠÑĥ дал +\ Security +- ar +Ġalleg ation +( Settings +ä nder +Ġell ipse +ĠRetro fit +Ġregul ating +ĠM olly +ĠL ok +_C ustom +ĠProm o +is in +Ġres umed +Ġmet ropolitan +.error Message +: ------------- +Ġpas ado +th ank +_De lete +ĠBright on +, unsigned +ä½ľ èĢħ +Ġaspir ations +-h ow +R ose += (( +_ne eded +_pl ural +< Application +ĠW EEK +ĠUn lock +ĠT EMP +S ou +Ġschizophren ia +Ġt roll +Ġcomplement ary +ĠNET WORK +Ġbl ir +Ġprogress Dialog +" %( +ĠAttribute Set +ĉ ts +.iter items +è¯ Ŀ +Ġesc rit +v ous +_pl aces +H K +Ġseg uir +_f w +ĠR ounded +Ġdis posit +è§ Ĩ +par m +w ow +STRU CTION +. allow +ĠChar Sequence +ĉ extern +Ġprosec uted +Ġmort ar +ĠJ uda +- msg +Ġest ud +.get Description +Ġs ow +amb re +Ġrom a +En h +bon us +Ġsqu at +Ġdist ra +ed Image +Ġpe ppers +-per formance +, ĊĊĊ +, file +ĠM IME +_con cat +AB S +-f ashion +Ġunder cover +One ToMany +Ġre claim +C OPY +Ġb inds +ĠT ape +Ġg ossip +ĠEqu ity +/ Card +. activ +' am +Ġdrain age +< Scalars +ĠonBind ViewHolder +() ?. +Ġs orrow +ĠI b +up y +_U UID +ĠCh arm +ĠElection s +.on Destroy +ĠInterest ingly +ounding Box +_d etection +-h eld +_ unknown +Ġrefr ain +Ġmét odo +Ġe Book +EN OMEM +Ġd ang +Prof essional +Ġd ictionaries +/m ysql +ĠST UD +Ġmas se +s cape +Ġdre i +: name +.log o +Sign Up +Ġt ahun +( theme +ĠFem me +Ġbom ber +ĠJ ade +ĠT ay +Ġsubmar ine +_cl ause +zy ch +Ġsimult aneous +Ġcas os +. boolean +(l hs +Ġcontin ental +-s ale +ĉ env +ĠC ute +ĠFactory Girl +ab us +/ value +Ġj adx +Ġst ern +> >ĊĊ +Ġsurf aced +Ġìł Ģìŀ¥ +pl atz +ĉ email +cept ors +"> ( +Ġep ile +è¯ » +ĠDe bt +åij Ĭ +N OP +" https +: j +Form Item +_L ICENSE +.get Double +ĠAg enda +ĉf inally +(f ilters +( av +ç¾ İ +AP ER +Ġl ava +еÑĢ ж +)) ))ĊĊ +Ġfault y +_n m +Ġtr ava +(B itmap +Ġspeed ing +> '). +Ġscreen ed +_ roll +ĠMac Book +ĠA UD +Ġdiagn ose +.G enerate +Ġ^ ^ +Ġstr s +[ Test +Ġr ansom +ĠDH CP +eld en +Ġinterpret ations +() ]. +flat Map +Ġline Height +_m ount +ĠW izards +Ġsl uts +eh ler +od al +Ġmilit ia +å ² +earn ed +Ġmis ery +int val +f und +Ġh ides +Ġdi arr +ĠWes ley +Ġx mm +Ġqu em +ĠAr abs +if th +ategor ized +Dis posable +P ure +_NOT IFY +sn ippet +ĠGar rett +.run ning +. weights +Ġ( -- +Ġin variant +äºĭ 件 +ĠAll owed +dir s +Ġpass ions +Ġl ad +ĠFl ush +men us +: block +Ġcompr a +.ch omp +alloc ator +Ġcur ated +ĠKnow ing +ĠPatt erson +Ġtel ah +' ex +Ġdo omed +Ġphil anth +ott y +.st yles +Own ed +Ġallerg ies += params +oc ese +it elist +ĠS ending +b ef +orr ar +ĠN ão +ĠF argo +ĠL ub +ĠComb ined +_g iven +ĉĉĉĉĉ ĠĠĠĠ +Ġreconc iliation +Pattern s +az ard +Ġbiom ass +ĠH ouses +resp uesta +cc o +/top ics +ĠY uk +Ġweaken ed +_c alendar +Ġmulher es +ĠMar l +Ġs ine +ĠT il +ĠSou ls +ĠDe utsche +ĠF OLLOW +Ġpip elines +ĠBever ly +_DIP SETTING +" # +ĠPro to +.b ig +ĠSav ings +ĠT anz +j un +ĠG amma +ĠS add +Ġadvis ors +Ġro ast +Ġun ters +ud ies +_l on +-point er +ĠElement Ref +\ Builder +example Input +.web driver +data Type +ĠQu ite +ĠCelt ics +u il +-def ense +b ish +ĠUI Window +ĠS uddenly +.h ot +.re ason +Ġg ör +AM D +.M ulti +auth enticated +reg ions +; ( +а ÑĢам +ĠKir by +$ route +PREC ATED +ĠDur ham +ow o +ĠPer forms +Ġdisreg ard +n st +ĠP ols +Ġget P +"] : +-col ored +( Keys +ĠAl leg +_mod ify +_ loading +str ained +Ġat roc +_p hr +< Sprite +Ġsatisf actory +m anship +.p ipeline +T ony +Ġth ief +pol ator +( lock +bur st +ĠOptim ization +Ġsurf ing +" Yes +Ġdesc ended +æ Ĵ +_C lear +Ġc ries +ĠFro zen +D IRECT +- Con +ĠLe icester +å¥ ³ +O OM += db +Ġget Message +< Student +_b atches +.M ask +_ eth +\ ) +Ġsom a +C atch +[ ch +Own ers +ind le +: auto +. vert +iv r +.set Location +Ġfl uent +_END IAN +ĠCar lo +cept s +add Action +.o auth +< UnityEngine +re ements +.S kip +? )ĊĊ +.default Props +Ġc abe +ĠSh en +eros is +ĠPro fit +Ġpo is +_C REATED +Ġremove From +(w s +? action +( Field +Ġerr one +.min imum +ĠRetrie ved +Ġd ado +ĠPR IVATE +-s pec +Ġg zip +p data +Ġpos Y +(l ow +Ġqual quer +/ cloud +ê² Į +( common +ĠAr beit +organ isation +Ġtid y +ĠRol and +( ph +.z one +Ġgent lemen +ượ c +å± ± +Ġenc losure +ĠMan afort +ĉ Color +St encil +N ic +Ġthe orem +ĠV G +Ġcol oured +V BoxLayout +uls ive +Drag on +c ff +et est +ens a +of day +.A zure +:UIControlEvent TouchUpInside +_up dates +Ġtrend y +ug as +weak Self +Ġr idge +ib ri +Ġì¶ Ķ +(C G +ĠMon key +.write Int +.tim edelta +ViewController Animated +ĠProvid ence +ãģ Ī +Ġbl ends +/Sub threshold +ĠAp pl +Ġat an +Ġreload Data +umb otron +st üt +O Auth +ĠG iving +ĠìĦ ¤ +ĠFinn ish +check ing +. Embed +sequ elize +Ġinitial izes +ĠOs lo +Ø ¶ +get Extension +_AL T +(bl ank +Ġfatal Error +Ġdem ise +**** *Ċ +ĠX S +(A F +ĠEn s +an tha +ĠP OR +Ġn ich +.N amed +Ġgig antic +ĠObserv atory +.Res olve +ĠPay ments +g uild +Ġcurrent State +============ ===Ċ +ĠS ey +p Data +Ġdead lines +Ġcentral ized +ĠScholar ship +_s upported +.ch rome +() ]);Ċ +Ġc yan +ĠC age +Auth ors +_ čĊ +/ os +k im +de e +.t ex +Ġyours elves +Ġm gr +Ġal k +-inst all +Ġdraft ing +Ġrum or +Ġstat ues +Pool ing +ol ina +AAAA AAAA +/* ---------------------------------------------------------------------------- +Ġextrem ists +Cal cul +ighth ouse +In set +(IN PUT +Ġsynchron ization +iv irus +. axes +ĠG ap +- An +_T emplate +Ġgam er +ĠCr icket +Ġl int +Ġauthor itarian +NS UInteger +Ġred o +Ġadip iscing +_F ETCH +che id +ĠF ang +. indices +t one +д ел +Ġ{{-- < +bra him +Ġsal a +get Code +Ġcommunic ated +start sWith +ert z +Read able +Item Id +oref errer +cred ible +á ria +Ġcombine Reducers +** /ĊĊ +Ġbl iss +Ġad orn +dep ends +ĠRO OM +Ġfr aming +Ġ? ', +aut y +_p ot +_t abs +Ex act +, ", +Ġ'} ';Ċ +Ġarbit r +ahr ain +.getString Extra +Ġ$ \ +Ġoutput Stream +Ġcomm enc +an us +ch y +< Employee +Ġhex atrigesimal +Ġn acional +(serial izers +_put char +_S AFE +ential Action +ItemSelected Listener +.Dis patch +Conf lict +_ about +os aur +Bound ary +Ġclear Color +( Location +ĠMON TH +ĠT aste +- General +ĠW AR +Ġer halten +-s aving +Ġcou pling +-tr igger +m otor +Ġy yyy +ĠPat ent +pt o +Ġmisdemean or +vas ion +ĠAdmir al +à¹ī า +_P WR +Ġdevast ated +fol ios +ITU DE +urre ct +Ġrobot ic +ĠSan ct +ĠHawai ian +.R oute +- condition +Ġr k +/**************************************************************************** Ċ +create Element +ĠK op +ign ant +. rollback +Ġsal ud +_ ', +ĠAN SI +Ex cept +ĠDraw able +.Utc Now +":[ {Ċ +Ġk ole +L ua +ĠBel ieve +Com put +Ġhall uc +ĠSign s +r st +.h u +ĠKN OW +W i +ĠBr ass +ĠR as +@ hotmail +Ġsed iment +Ġap k +Ġì ĥģ +_reg ions +Ġpod ium +< Book +ж е +Ġsix teen +ĠAli as +Ġinfr ared +ĠV ander +ĠLe ading +uc ing +,: ,: +_h or +w at +Ġdé cou +_W idget +S ounds +_n avigation +Ġschn ell +(g enerator +uc ene +Ġrem ake +IP v +Ġré al +_IN CREMENT +Ġhypoth etical +_ ang +Ġof s +Ġ! Ċ +.com pleted +Get Type +Ġkom men +ál ido +add On +Ġz ÅĤ +UL A +_ind icator +'] ĊĊĊ +ap ache +_S elect +ĠGre ene +Wh ats +_an im +Ġrepet itive +m uch +ĠTh reshold +Ġl f +(C ategory +con e +M ix +_MET ADATA +ays ia +Ne ighbors +ĉĊ ĉĉĊ +IP HER +ĠFr ag +ĠC ells +Ġnames paces +( back +ĠRest aurants +sv c +Ġл и +ote ch +-s l +¥ ¿ +ĠW T +ĠRed uction +Ġd otted +ĉf ound +ĠTE AM +B orn +ĠM ush +ĠCompar able +Ġh itch +AT O +Ġmax Height +begin Transaction +ÃŃ v +_b n +Ġher d +Ġrevers al +ĠH ond +del imiter +Ġconf use +Ġh ops +Ġcent roid +Ġcourt room +.decor ators +Ġm pi +ĠImpro ved +IN NER +ĠBang alore +ĠT amb +Ġbo ast +() ))čĊ +Ġil licit +ĠMor occo +greg ator +_res ume +Ġcrack down +Ġport raits +/h igh +( \' +Ġay ud +_fe edback +Ġc ate +/ avatar +Ġhe b +Point Cloud +Ġå ĴĮ +Ġ< ![ +Ġget Resources +} :{ +Oper ating +ĠF og +ĉt ab +ĠResearch ers +Ġfabric ation +.datas ets +ĠCamp o +ĠKa uf +Ġd ll +lig t +] ));ĊĊ +st ellen +ACK ET +l vl +ĠGl ory +.date Time +Ġcomm ute +ĠonCreate ViewHolder +ĠX Element +ĠT okens +< thead +_p ick +ì ¤ +v on +depart ure +(render er +phone Number +(P erson +gen es +ĠL ars +Ġ) {ĊĊ +ĠJson Result +Ġmet odo +VO KE +.get UserId +Acc eler +ĉ required +Ġchampionship s +Build Context +/t ask +/re leases +C ategoria +_over lay +Ġscar ce +_l im +n gr +ah len +ĠArt ificial +sp read +Ġbow ling +.an alysis +SM TP +ĉp assword +Ġbath s +] )){Ċ +current ly +ac iente +_se parator +Ġde ber +ĠDis abled +i ères +Ġâ ķ +_process ing +Ġprotest ing +ĠR OT +gr ab +Ġз ак +Ġpro active +word press +ĠSe ver +ind en +Ġw ikipedia +){ čĊčĊ +_w indows +is lation +Ġun rest +Ġdismiss al +.N UM +_F AST +iss ued +ĠF ACE +_u nder +Ġpl ugged +Ġå ° +ĠbÄĻd zie +ĠI CC +Ġcombust ion +Ġkiss ed +Ġstar red +ĠW atts +Ġspi elen +-p urpose +ĠE val +arg es +, result +techn ology +Ġnational ity +ic us +ĠN ug +ĠÑĤ о +ĉĉĉĉĉĉĉ ĠĠ +col o +Ġg astro +ante ed +OL ID +.b ias +_t ele +.ins pect +Ġve il +. footer +Ġneglig ence +Ġjud gments +Room s +yn n +ĉcount er +occup ation +Ġ çĶŁ +un as +Ġ(^ )( +L ambda +f el +.Param s +Ġд обав +set Layout +Ġdeport ation +Ġlocal Object +ĠPharm aceutical +cept ive +ĠN ome +Equ ipment +F an +Un iversal +ĉ socket +Ġgr in +Ġex poses +Ġhab er +Ġsincer ely +Ġc ams +Ġm ü +en ia +E mer +C rypto +Sl ow +(x hr +! =( +-s ervices +ĠP W +Ġprend re +Ġm ädchen +em ons +озв ÑĢаÑī +.M anager +ì Ļ +Ġg raf +- ra +met rical +/ fl +Ġc emetery +g ens +Ġp ÅĻ +ĠMySql Command +- To +Ġv Ã¥ +Ġa irst +oment um +Ġserv o +m illion +ĠMir anda +" She +Ġadvoc ating +-c aption +ĠAt tribution +Ġwel che +_v endor +ĉ Status +arr is +Ġprint k +"," # +Ġrel ativ +if ferences +izz es +Ġdec imals +ĠPro v +.max imum +Ar n +Ġhelicopt ers +_B OTTOM +ch ure +od ings +' ( +")) );čĊ +( bean +.f d +F und +Ġhang s +app id +/k ernel +.p oi +.Min Value +- validation +L uke +c df +ĠFun eral +ĠS amples +ĉ de +Ġto astr +Ġtax able +Ġcl ustering +Ġ'\ ' +Ġre straint +ec ed +ch ains +ãĢĤ ï¼Ī +_GR APH +Ġfue led +éľ Ģ +H p +å¤ į +T iles +Ġa unque +J C +Ġhost age +ĠE sk +Ġm av +Ġgest ion +Ġb anners +} {$ +.int Value +.' "ĊĊ +_M ATRIX +Ġce ased +ĠG OD +_CAM ERA +.Allow User +tr acked +C ook +b airro +( company +Ġview point +.get Writer +ĠN ets +w ives +Ġ( ))Ċ +example Modal +ĉ child +Ġmyth ology +Ġ// " +_ axes +ib old +.D ark +ĠMax well +Ġg pointer +olic itud +B at +ul ner +bal anced +mail er +Ġcont empor +æīĭ æľº +(" __ +Ġ" )" +re ar +ĠHu ang +] ')Ċ +× © +FT A +ĠCalling Convention +ĠOutput s +P k +.Re ference +lect ual +Ġ) :ĊĊ +Ġbrace let +ug er +ĉ Error +S weet +("/ ");Ċ +h x +Ġun reasonable +Inter preter +Ġlo ft +_product o +Ġsoci etal +.P arser +ĠAd apt +. foo +( where +.F eature +ĠYam aha +g lass +For ge +Ġprohib its +Ġcapac ities +Ġíķ¨ ìĪĺ +Ġper mutation +Ġih m +F ld +el ial +======== ===Ċ +@ Configuration +Ġge ared +ios o +iest a +trans lations +Input Change +Pop ular +ĠPL US +Ġv f +_F ree +b box +Ġcaus al +PI LE +Ġsch ö +Ġiron ic +M ir +. @ +åį Ĺ +Ġè ĩ +R ew +ul ence +fl en +Ġcan Activate +- response +Ġacc ents +ign ored +° F +.Dependency Injection +ĉ point +Ġconting ent +Ġsqu ash +Ġpar ms +ĠC emetery +Ġdelta Time +ĠD OS +Ġvan ished +аÑĢам еÑĤ +ĠD PS +t foot +ĠZ us +_IN STALL +G AN +Ġar b +Ġmunicipal ities +Into Constraints +AutoresizingMask IntoConstraints +, image +_ ignore +Ġdanger ously +quis a +pl uck +Ġhar us +up pe +Http Exception +Br acket +.' 'ĊĊ +ĠT ol +ĠView er +zb ollah +.Code Analysis +ì nh +Ġcorrect amente +.d a +ĠAl ger +× IJ +ba um +ĠPan ther +part icipant +å¿ ħ +-s up +Ġem ulator +Ġf ading +ĠW olver +cre ates +Ġbook ings +.Q uestion +§ è¡Į +Ġstress es +Ġre written +.PI PE +ed es +Ġc bd +": "/ +Ġenh ancements +_s y +B IN +ĠSl ip +Ins pect +ĠW eg +Ġcon gregation +Ġ_ : +_r m +Frame buffer +Ġ'& # +ĠFall out +Is Required +ĠPear son +ĠF ACT +Ġrel ie +ĉ box +ĠShe pherd +ĠWiki Leaks +ĠCollect or +Ġres ized +method Name +Ġevent Type +ĠA then +Des criptors +Ġb ers +- oper +ĠInitial ly +å ¡ +_B TN +ĠĠĠĠĠĠĠĠĠ čĊ +á b +_c ampaign +_w atch +F ord +-date picker +Ġvis c +Ġsat u +_s ms +Ġcont ador +-s vg +ĠDO I +$ args +Ġkn ob +.B OLD +Ġdeb ated +img s +sock opt +tr uth +ĠFe es +Ġh Wnd +_f ood +Ġab ras +Ġnot ions +ĠT od +: create +ĠConf lict +Us uarios +OT OS +Ġm sm +K HTML +([ ( +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġ} ] +w izard +Ġm ientras +Ġdata List +Ġemerg es +Äĥ ng +.Read Int +PG A +ILL ISE +I Enumerator +(t uple +Christ mas +Look AndFeel +og enerated +Ġ# ĊĊ +control led +Ġex quisite +Ġa cest +Read Write +G ain +ãĢį ãĢĮ +Ġcopyright ed +Ġdo om +.Table LayoutPanel +ĠD ort +Ġch ili +Ġwer k +ĠEVENT S +ĠBe acon +Ġship ments +Ġse bagai +up on +ut om +.con verter +.Drop Table +={ }Ċ +f ic +~ ĊĊ +Ġlesb ians +_n a +Fore ign +ĉ then +/ ms +Ġor i +get Property +ĉsn printf +hes ion +ãģ ¤ +"} ," +Ġac rylic +P ers +@ Enable +I sl +(C ard +. Stack +L icensed +_G UID +: title +Ġh ust +Ġprincipal Table +an itize +/ embed +Ġens ured +ĠE GL +ÙĪ ر +ĠåĪ Ĩ +/ ,Ċ +Ġfundra iser +Key Name +Ġmarch ed +_VAL UES +ĠSc enario +Ġmet ic +_ass oci +ĠPast or +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉĉĉ +er ate +Ġinv itations +quo ise +Ġbl aming +Ġd aring +UM MY +Ġrich er +em aker +ĠIdent ification +ĠìĿ ¸ +ĠBinding Flags +ch as +Ġresil ient +_p g +Ġre leg +ĠI RA +ST E +Ġtr actor +- loading +ĠPre viously +ĠV acc +/ be +Ġn Ã¥r +Ġurl encode +ĠNor folk +.Re lease +ĠNe utral +ä¸Ń åĽ½ +ĠAr lington +Ġalleg es +ĠW riters +Test er +ĠR ally +Ġc á +ĉ Print +Ġâĩ Ĵ +ĠUser Controller +ĠSeek ing +.V AL +List Node +_ ff +ĠPhill ip +FA CT +Ġc aramel +ĠM ultip +ĠCom pared +ĠSer bia +Ł ³ +Ġrev ive +ĠK anye +Ġver ge +ĠBulg aria +get Body +Ġ| > +ce ph +.DateTime Picker +." ;ĊĊ +ĠT ie +, item +Ġm enn +G as +och a +_v irtual +Ġmaster piece +_se quences +L TE +ĠSub mission +Call er +$ \ +S port +ag us +Constraint Maker +Ġcol oc +Ġw ig +ĠÐ £ +ĉ Array +Look s +ĠGT A +.st eps +atch ewan +_r anges +ext Alignment +ĠBren nan +Ġab straction +uler Angles +.m isc +Ġantib odies +Ġexponent ial +ĠCH ANNEL +exp ense +' y +Ġdetect ives +Ġpur ported +Y STEM +Ġradio active +ĠLat ina +.Enc oding +.T AG +x in +D egree +ur acion +pr ices +ĠRefer entialAction +Ġr arity +Ġp iles +g ende +_project s +_g lobals +.start Time +Ġê µ¬ +SE CTION +_p ublish +F ault +DD L +_p rior +M om +Ġth icker +Ġsequ elize +Ġessential s +str as +in tr +>( () +.man agement +e il +éĹ Ń +A ware +.C ity +ĠAr bit +_D M +_key board +L Object +- webpack +ĠNew port +Ġprincipal Column +leg ant +Ġp allet +Ġfract ure +Ġg mail +.M eta +A bove +.Key Event +j it +_mac ro +_P USH +á» © +/ controller +åĬł è½½ +Ġsuperf icial +exter ity +Ġmens agem +W ind +ist on +.open api +и ÑĢов +ĠSerial izer +uct ive +Ġz ar +Pl aces +.St atic +B a +Ġin advert +ĠIndones ian +_IP V +(h orizontal +Ġget Title +ide press +ĠConsole Color +ip ers +$ out +Ġfest ive +Ġeven ings +.Get Data +uit ka +ĠManual s +uss ed +_M ax +.Ch at +ĠA ircraft += com +FO UND +ap ro +Ġtre asures +_al ive +Ġgad get +ek ing +Button Down +B rowsable +.PER MISSION +P ASSWORD +ĠH ASH +f é +\ TestCase +LO SS +o thers +, J +Ġassh ole +wer k +Ġm ã +. ie +ev il +kont akte +//////////////////////////////////////////////////////////////////////////////// Ċ += sys +ĉ lock +-- ;ĊĊ +_F UN +Fill Color +ó a +pre nd +Ġcompress or +M other +ĠAr cher +.g oto +Ġwür de +Ġbam boo +ï¼ İ +ĠT rees +Ġb umper +Ġsa usage +ĠEl asticsearch +Ġhor izontally +ĠG ul +Im mutable +Ġlos er +Ġabort ed +-d emo +ĠH atch +Ġund e +Ġprocess o +-c all +In come +å ĥ +_ returns +']." ' +(s w +C BS +am ilies +ĠYour self +ĠH olt +.M ON +ৠĩ +ÑĪ е +an on +ĠFont Awesome +produ cer +j r +Ġm au +ĉint er +Ġdish onest +Ġmagn a +ĠCollect ive +Ġvra iment +Ġcho ix +st ay +Ġweld ing +r ising +, min +ĠF ate +g lob +RGB A +Ġdet te +V en +Ġembarrass ment +.DE LETE +greg ar +-re nder +(b ucket +"> ĊĊĊ +.wait Key +Bus y +Ġdifferent iation +ĠC ST +.Con stant +Ġline Number +(m atches +Ġweb socket +Ġbar red +Ġpued es +M ono +C ORE +I ID +ĠĠĠĠ čĊčĊ +Ġpúb lico +lean ing +Ġcleans ing +Ġcr is +ĠDev ils +_SET TING +unt ary +. );Ċ +Ċ ĠĠĠĊ +[ curr +ts y +ĠAlex is +rit el +Ġpet roleum +.pre processing +m atter +For Result +- license +Ġtrav ellers +ĠDispatch er +enn ifer +Ġdigest ive +P ED +hib ition +MAS ConstraintMaker +ĠW att +Ben ef +.set View +d to +TE E +ĠPel osi +_EX TRA +Ġmed als +x hr +fore cast +Ġn argin +oun s +-f ill +_CUR SOR +Ġsuperv ised +Ġtur f +ĠEd gar +POS ITION +Ġcategory Id +â ī +_ ER +ủ a +Sh own +. ll +_POL ICY +(), ' +ĠPre v +ĠString Field +ĉG lobal +ass ed +Through out +o stringstream +.awt extra +Ġslo pes +ĠSe quential +Ġgi orn +Ġz elf +Ġvers atility +lene ck +.c gi +Ġdou bling +ĠBang kok +Ġbu urt +Ġusu ário +st udio +Ġje unes +Ġm uted +Ġ ips +_f raction +&& ( +Ġst unt +'); ?>čĊ +Ġev apor +b able +ĠPR ICE +Ġæ ³ +lu cent +Ġv amp +ĠTechn ician +Ġuniqu eness +M es +ur ban +.param etrize +ĠRe play +S essions +em br +-Americ ans +_PRO XY +Ġp ian +Ġtri e +ĠD estructor +Game State +ĠIM F +ch in +Ġport e +ĠSw al +åŁ İ +Sub string +im ing +/L ibrary +Ġfright ened +w rites +Ġrecurs os +ar Result +_INIT IALIZ +ĠBad ge +_c rc +E ight +ĠDIST INCT +Ġth ro +@ Xml +ĠLegend ary +-t witter +_e asy +Ġ+ ++ +(D ATA +.L ocale +Ġk ä +Ġn urt +Ġcr uis +_ ios +Ġsens ing +_L ine +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +pon g +ole on +Ġwild card +çĶ¨æĪ· åIJį +Ġbeg ging +R od +ĠÃ İ +_C ELL +Research ers +. selector +_ ing +Ġaspir ing +Ġimm ortal +Ġy min +_ robot +Ġpl ur +B TC +ĠD ID +Ġpier cing +* u +_DEFIN ED +ĠTh i +ita ire +(m edia +- ons +Ġche fs +Ġ"* . +/ AP +Ġraz or +Ġsearch Data +Ġ= & +Ġ ãĢĤ +Ġm ourn +ting ham +Ġo li +ĠVern on +_R S +ŀ æĢ§ +Ġf ácil +ang en +cel ain +Ġa il +le st +ĠQ COMPARE +g ain +ĠÎ µ +ĠK ob +ĠF ault +_config s +ç»ĵ æŀľ +. + +cal ar +(color s +M ul +_ ART +Ġexperiment ing +erm en +ĠAng lo +.Fixed Single +Se a +Ġc txt +.s lider +C ollapse +G rey +Ġf ld +-pro of +.cap acity +get Parent +ĠCom pliance +Ġburg l +- rec +Ġover written +M U +Ġrout ers +ĉ Model +Ġfantas ies +av ian +_p rec +ĠSc andin +Ġ// < +/o ct +Ġceremon ies +Month s +und y +Ġqu ed +ĠN ou +ĠV ibr +.r gb +Ġcit rus +Ġbr aces +-upper case +get Table +Ġdop o +ĠK err +_CH ILD +- cloud +ĉ Matrix +Ġgard ening +S ing +al most +Require ments +ugu ay +( Property +sub scriber +FA ST +re action +(l p +) })Ċ +` ). +.w allet +_ex change +.Max imum +ĠVer b +âĶ ģ +() < +ï¼Ľ Ċ +RO T +C ARD +ub it +{ @ +_k el +ĠTool tip +My SQL +Main Activity +ar f +Ġm align +Ġse inen +ap ist +Ġ< % +Method Impl +M il +ĠM ick +.de pend +< ID +Ġpredict ive +ĠAP PLICATION +le f +dim ensions +Ġconoc er +/ conf +ĠTr acy +F oto +_rem aining += file +Ġpage Index +ĠPar ish +Ġt exas +ĠM AGIC +ĠH ew +d ifference +Ġalt ura +c um +ĉdata Type +Ġcaracter es +avi ours +ĠV OID +è¿ ij +P UBLIC +B io +ĠstringBy Appending +Parse Exception +ĠS uff +ĠN orton +/d etails +.n ull +>> & +ĉ ok +-l ow +. usuario +n ested +X B +OUR S +.Border Color +Ġb row +ĠÐ ķ +cor r +ĠRed skins +.get Tag +.get Transaction +Ġst igma +hard t +ĠPlayer Prefs +als y +uc son +L anguages +ĠOl ivia +Ġt ac +Ġb li +Ġc aval +Ġconsolid ated +Ġper il +Ġde le +Ġform ulated +Ġhigh ways +.sp awn +== $ +ĠN iet +Ġv eggies +yp o +-r ule +ĠV ie +/e pl +Ġenf ants +string Literal +Ġtou ghest +buy er +Ġcov ariance +Ġil i +ĠSoph ie +ĠB AB +Ġ" ), +ĠU k +current Index +_user data +.code c +ĠPun jab +ĠSN P +l ol +adv ance +Ġcom fy +Json Ignore +Ġfashion able +ĠI CON +Ġor a +ĠP ricing +< num +ĠI RC +ER V +ĠMe in +ĠID ictionary +AD OW +is New +ĠDev on +at l +(request Code +ĉ PreparedStatement +IM PORT +Ġmar ital +_SELECT ED +get Response +ar Down +B V +ib Name +ĠP ATCH +ä än +Ġda ar +ĠFile Mode +Ġm arty +.Spring Application +c ene +amp oline +get Size +Rest art +æķ Ī +.project s +ĠEthi opia +Ġstatus es +T ION +(b g +ĠX unit +Temp orary +ĠEng agement +Ġx f +Ġprox ies +Ġgen esis +Pager Adapter +ĠSl ave +Ġsung lasses +ĠCh loe +Ġko ji +ad em +ĉ JSONObject +Î ³ +Ġh ors +* w +ó r +es ch +Ġcritic ised +z ial +ĠSale m +.Vert ical +ĠR ash +> E +ter ing +/s creens +Ġheight ened +аÑĢ ÑĤ +Author ities +_b box +ün st +.font Size +ĠBO OLEAN +div ide +ĠSlo ven +uc er +Ù Ĵ +st ub +Ġnavig ating +: animated +_N OW +_v ect +} {Ċ +@ ( +Ġtele com +Ġcontract ing +ĠAss ange +Ġextract ing +Ġgr ö +c obra +.D IS +Ġcr ab +Ġtw itch +Ġvert s +Ġreject s +ĉ format +Ġreg eneration +.S ys +s olve +ĉd ialog +sh i +m eter +(b est +valid ators +Ġon wards +Ġg uru +Ġmoder ator +ow ied +ex periment +r ub +Ġm qtt +ĠCa ucas +Ġnational ism +Ġm ange +ĉ ImGui +/ Edit +Ġin h +Ġint ellig +ero kee +ĉ export +Ġdiscrim inate +sub tract +ĠM oodle +ens er +ĠGuid es +R AP +-h ot +_gr p +.p icture +X A +Ġinit View +_Com m +Ġoverd ose +Ġ+ ĊĊ +ĠSil ent +show s +Ġinterpol ate +Form ation +Ġb isc +mark ets +( SC +Z e +ĠNetwork ing +Ġad renal +ĠG uns +ete or +Decl ared +orget own +Ġk arena +/ password +_address es +ITER AL +B uzz +ĠCon way +(c ase +P WD +he iro +( act +** čĊ +());ĊĊ Ċ +Ġan v +Ġ. .ĊĊ +(Menu Item +(m ail +_section s +ĉ net +Ġpl ut +Ġw rench +/ object +ĠI st +ĠV IS +/p ub +al ten +Ġguit ars +Ġantibiot ic +ï¼ ĸ + ¹ +Ġ" +" +form ula +Ġbab es +ĠP rompt +Ġen im +/ player +ĉ ref +Ġby Äĩ +Ġconsum es +ĠH ast +ĠT ao +Ġ' ))Ċ +Ġcl am +Ġthigh s +Ġmot if +Api Operation +ĠW L +get C +ĉf lags +oint ments +Ġeconom ical +need le +x ls +pr actice +ut zer +time ofday +- output +Ġfind ById +ĠBudd y +Ðŀ ÑĤ +Se ven +ĠB ark +Ġenv oy +_al gorithm +åĪ © +Ġball istic +ç§ » +r ades +ĉd oc +rodu cing +ĠE ating +Un mount +/data Tables +_b onus +Ġl itt +pp s +) localObject +per f +ĠHel vetica +sh utdown +/ ml +.t okens +ĠHard core +, row +/b g +Sc aler +âĢĶ as +_log its +âĢĻ int +ĉ App +Imp licit +.F printf +ET O +Ġterr a +Ġpossess ing +.r strip +, ), += yes +ĠStr ipe +? = +ne utral +.g ood +Ġk ennen +ĠS ung +f ault +ystate change +Can adian +',' ".$ +ĠM its +æ nd +ĠSTR UCT +ĠURL WithString +ĠCom pass +Ġ-- ĊĊ +ĠNS LayoutConstraint +| min +-ad just +Ġreb uilt +L IGHT +/ se +-m ount +vp n +valid ated +(Q Object +Ġign ition +ĠCharg ers +RYPT O +]initWith Frame +ĠFl uid +Ġcad re +Ġnomin ations +Ne ill +ĠH ou +Ġcurrent s +_g ene +(in p +Par is +z ÄĻ +ag gregate +Ġass oc +weet ed +err at +âĢĵ ĊĊ +Ġ'/ ',Ċ +fix ture +ĠH ighest +amb ient +Ġch mod +Ġcon te +Ġsens ual +Ġgar ment +z ers +ĠPower ed +dom ains +R eward +i omanip +Ġcock pit +out file +Ġbuilt in +Ġins isting +. vars +zip code +Ġ ���� +f ails +Ġconsolid ation +_ oid +Plan et +Ġ= ", +ĉ el +UIL T +ät z +af ari +ĠMc Cl +Tim eline +Est a +Ġfr am +Y E +Ġcere bral +Of Month +ĠP regn +Ġкл аÑģÑģ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +ĠF res +Appro ved +.S pecial +ĠProtest ant +Ġallerg y +_p cm +ĉC opyright +Ġsuper Class +" strconv +ĠMoh amed +Ġ' // +Fore Color +Ar thur +ĠJ ungle +Ġve ins +S ad +Ġback ups +ĠOp inion +û t +Ġinter mitt +ody n +ĠChrist ina +Ġand re +Ġevac uation +pa lette +h orse +ĠRes ident +ĠHass an +.N il +Ġa isle +ĠG rowing +Ġblog info +/s ql +_io ctl +Sc aling +ĠMon ad +_c pp +ĠH utch +ĠApple WebKit +Exp ense +_J OB +Ġpoint less +From Body +ant al +Ġdepict ing +ĠC ELL +Ġref in +ĠC NC +ì¹ ĺ +_dim ensions +ĠS AN +Ġa ft +Ġfoot steps +cc oli +_PH ONE +/m ath +-k ind +ĠMe ans +ich ael +.g una +Ġinaug uration +-dr iving +( delete +Ġtotal Count +_M C +.Ext ension +Com mercial +Ġz Index +< Customer +" g +-sh are +Ġp act +ag ara +ĠS IL +_m odes +ĠM olecular +Ġsystem atically +< G +_s cr +ĠO ro +as ers +Ġb ic +Ġdest roys +PI PE +.Start Position +Ġc ủa +ire z +.B unifu +_F unction +Ġs ü +_f uture +ĠWe alth +ĠNatur ally +æĢ » +_y es +Ġabrupt ly +String Encoding +ĠCGPoint Make +Ġz h +Ġimp erson +Ġpiv otal +ĠSom alia +Ġsegment ation +_AN AL +ĠLogin Component +Cons ult +Ġtr uncated +] ";Ċ +.get Config +Ġintern ship +B aby +ê° ľ +Ġstrengthen ed +_M I +b asket +Ġnicht s +ĠTV s +ĠSh an +ãĤ µ +rac use +.Re LU +/ interfaces +ĠgetItem Count +Ġret iring +Ġspecial s +Ġentity Manager +bel ief +Ġs older +da ughter +ij kl +Ġutil izes +.f ixed +S U +Ġdr astic +Ġh acks +gr und +ĠM U +ĠSt arter +.Com ponents +_m otor +Gold en +Ġl odge +Ġ )); +ĠCor inth +иÑĩ еÑģÑĤво +ón ico +gre SQL +ĠFl uent +Ġmar c +.Load Scene +.Group s +Ġer h +ĠAut umn +St opped +Ġitalian o +Ġmin ions +ĠAssert ions +Ġm ux +B u +Ġ---------------------------------------------------------------- -------------------------------- +ĉ up +read ystatechange +_M eta +Ġcurrent Date +ĠChap man +Und o +Se an +ap r +Ġpar m +_ icons +ĠSt a +á z +Ġsub division +Ġalter ing +P NG +ponent ial +Ġpost gres +ĠB DS +-ex istent +ĠBrad ford +ĠO MX +_W HITE +_PRO GRAM +q c +Ġtypings Slinky +ĠP ics +_M ETA +IT TER +_sub scription +IRON MENT +ĠHy undai +();ĊĊ ĊĊ +ĠØ ³ +Ġj ac +Ġelimin ates +) });Ċ +Ġcomp rend +ĉ insert +_f aces +"> $ +Ġeb ay +Ġcapt ive +pl iant +ĠCalcul ates +ol ta +est ing +_re vision +Ġm ús ++ m +"," "," +WH AT +Ġcompassion ate +h arga +[ random +Ġmod ulo +(s n +Ġoccup ations +//// Ċ +ĉ board +ĠB alk +wi Äħ +ĠW ifi +.Pro file +:m aj +ĉm at +LOCK S +(j Button +Ġ(' $ +M ur +æĮ ī +b ble +Ġf rog +-h ide +Ġbroad caster +ภŀ +ha led +Ġam using +_predict ions +_in tr +Ġe agle +аÑĤ елÑĮ +Ġget List +ps ilon +Ġcharacter ization +AR DS +Ġre location +Ġr ulers +P AY +ĠDef initely +_A ction +Ġclos ures +Ġfact ual +odyn amic +Ġpreca utions +nie j +ĠPart ies +ĠSub aru +Ġcous ins +ar beit +.m oney +gun ta +( and +get item +.Style Priority +Ġsl id +single ton +Ġg arn +ĠP AS +Ġd azz +a ż +Ġbog us +ĠM og +Ġrival ry +is ol +Ġland marks +ñ as +B ern +ĠSach s +Ġ" )ĊĊ +Ġhost ility +_m ex +m ere +M ot +p ictureBox +Def ense +Ġaffid avit +other wise +.d irectory +_ UnityEngine +-b log +.s kin +ph em +Ap ellido +er chant +[ class +Ġw art +." [ +ale ur +/ back +ĠĠĠĠ ĉĠĠĠ +Ġprecip itation +Ġob struction +Ġp Obj +Ġr upt +UCK ET +ay e +æİ Ĵ +g x +Ġe cl +Ġsecre cy +/ Header +ĠLes b +Ġle i +ĠBullet in +Ġgive away +.H ome +_RO OM +" W +Ġcow ork +_ ra +ĠC ycling +ĠP aw +Ġpup il +/ arch +ĠFile Utils +é¦ ĸ +r sp +Ġfreed oms +ĠL ear +}` ). +Ġbow ls +/b lock +_log ging +Ġmeth ane +Ġhorn s +Ġwonder fully +Ġalter ations +Ġex ile +ls en +_p ause +_L ANGUAGE +ĠUS DA +_m ysql +_AM OUNT +ĠL IFE +Ġyoung sters +Ġri ots +[ E +Ġun forgettable +, },Ċ +Dis posed +ĠAss assin +UN G +ĠNew sp +User Service +: aload ++ ', +Ġsett lers +Ġscre ams +Ġincon venience +.R otate +Ġj ars +ĠP uzzle +Ġm est +ars i +ĠSh arma +| ( +.d s +ĠSac red +_e vt +Ġexpress es +Ġh och +ĠD uch +.c alls +th r +ĠShe ffield +.Alert Dialog +Ġrad ically +Ġtr ous +Ġprev ailing +ĠWW II +âĢĻ n +ens ely +ĠY esterday +ĠSir ius +Ġkill ers +ĠF FT +Ġo val +') :čĊ +Ġìłķ ë³´ +our age +ĠCheck box +Work book +.def er +_f loor +Ġc ouncill +Ġnors ke +mo il +ore a +Ġmarket ed +_S UR +x AA +Ġst ained +e ut +ĠM eng +Ġi eee +. extern +eg ie +Ġr app +ĠPy ongyang +' class +M ob +Ġinitial Value +_w ave +Ġj ab +Ġmascul ine +Ġampl ifier +Ġt ty +Path Component +_ xt +ĠG FP +/ sec +ĉdis patch +mark down +ĠS chn +bo le +· · +mouse move +Ġerr Msg +Ġas ign +_m ono +To Selector +ĠZ u +(R ect +ĠError Code +lat in +ang ible +v tk +CG Size +P okemon +Ġclass mates +Ġattract s +ĠT atto +ult an +ol óg +Ġhalt ed +ठ¨ +ĠK art +Ġ ue +_Init Structure +Test Class +ĠAir bnb +_ ", +Ġchar coal +Ġip c +ĠSt retch +.g lide +lates AutoresizingMaskIntoConstraints +Ġpot ion +ITT LE +Ġcount ert +_h d +pre pared +Ad s +ĠV ampire +rob ots +.Create Index +Status Label +Ġt ucked +af ür +U t +Ġswe ater +_F N +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +ata ka +Ġeyeb rows +ac oes +ud en +.LinearLayout Manager +Ġsw ay +Ġmult in +() )))Ċ +ĠNS UInteger +ĠMy Base +Part ner +uts chen +ĠC ater +.setBackground Color +Ġaccompl ishment +_pro blem +.d td +Ġpage Number +Ġj ackets +Ġcro pped +u els +ĠH ep +Ġc apped +* Math +_callback s +Ġpub b +ĠBrun swick +.res pond +[" _ +Ġbed ding +hyth m +O X +(s peed +Ġpestic ides +Ġ---- --- +.Bl ue +Ġnood les +ĠGo es +Ġs aver +o xy +_com pletion +ĠSw inger +Ġget Date +Ġmind ed +int egration +ĠLot us +(st op +(', ');Ċ +Ġflood s +ĠWork flow +Ġerupt ed +Mac ro +ĠSau ce +Ġevent Name +\ Input +Break ing +ĉ when +_p w +IND ER +ĠWell ness +Ġvox el +ĠM ell +ĠM EDIA +SE NS +ĠFund s +ĠM ild +< Array +- this +ump ed +/f w +ĠDb Context +W I +girl s +H OW +'); ?>Ċ +Ġtempt ing +Ġtest ament +Ġb ible +Ġconsult ed +ĠIndex Error +è¨ ĺ +Ġkey pad +izz o +( ok +Ġwhats app +ĠRemote Exception +Ġteam ed +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +» , +Ġget Time +di ag +iss y +Ġh ed +Ġkn ots +j om +Ġfun nel +-m ails +Ġexport ing +ĠV L +ĠK arn +ĠBuddh ism +ĠAll an +_R ADIUS +Ġw ording +ĠFor get +ĠCor ona +ip hy +Ġlim burg +ugg y +ĠUser Repository +im in +(e le +Ġlabel led +ç¤ ¾ +ĠH erman +.q q +Ġ" ));Ċ +ie ber +.Trans late +ry n +Ġdes env +um d +Sim ply +ĉm ode +R pc +ĠVal encia +Ġstaff ers +Ġsel v +ĠSpi ke +Ġdel ic +Ġer u +_D T +J udge +á» ķ +ĠBas in +.m utable +" url +Ġtar iff +ĠSlee ve +Ġfl are +.drop out +Ġbr ides +)) ,čĊ +_con straints +de struct +Out line +Ġdisappe ars +_lock ed +ĠNS LocalizedString +ck e +ĉ null +ad resse +Ġto pping +ĠJ oker +b ishop +но ÑģÑĤÑĮ +and ering +_ amp += time +_S pace +_P ULL +' = +Ġant iqu +Ġc ach +___ ĊĊ +ON ES +о Ñı +Ġun read +.p olicy +oooo oooo +ëŁ ¬ +Ġu sted +ĠRe ce +Ġal lem +ãĥ¼ ãĤ¹ +ĠThought s +ve illance +istr ate +_l ane +Ġfam ed +.Get Name +Ġsmo other +ĠQual ified +az ers +_ geo +F ax +ĠM inds +ĠR aises +Ġtrans cripts +Con versation +Ġremark ed +ëĤ ĺ +d ling +Ġdeploy ing +Ġshared Application +Ġk p +FontAwesome Icon +_d ummy +reib en +ĠJane iro +Direction s +.get Bean +s ass +Ġcommand ers +v ation +error Code +ĠAl loy +.local ized +Ð ij +Ġdish washer +ĠSou p +N u +_D efault +Ġune ven +Ġ/> ";Ċ +-B ased +Ġseam lessly +- null +ĠX C +Ġst ew +(d elay +AT ORS +ĠWhe eler +" H +e ast +. air +âĢľ But +Object Context +success fully +_l and +Ġfold s +_CO ORD +Ġsub po +.get Address +in str +Material s +Ñĥ ÑģÑĤ +de posit +-l ast +_GR AY += find +Ġmut ant +Ġlesb ienne +let cher +RO UGH +ure ka +.c apture +Ġen n +Ġ([ [ +ĠFl u +Ġtask Id +ĠHus sein +.f older +Ġa usterity +ISTR ATION +_ Impl +注 æĦı +Ġdec ree +- chat +Ġimp lication +Ġguess es +ul kan +An alytics +. plus +COM MAND +е ли +» ĊĊ +_S ITE +Ġequal To +Support FragmentManager +ĠRec ording +å®Į æĪIJ +Ġbag gage +Ġpitch ers +ĠE h +o que +ĉc nt +Ġ=> $ +/ foo +IR A +ĠSat ellite +bor ah +Ġ}} "Ċ +ĠEnd s +ĠSpr ay +, param +.Ch rome +* q +th ought +ibr ated +Ġth ieves +Ġbenefici aries +Enter ed +ottes ville +Ġveter in +By ID +qu ipe +um ption +- unit +Execution Context +@ s +ĠG iov +.Tool Tip +_f riend +( attributes +Ġdump ing +ĠJ C +_D OCUMENT +ĠArm our +( insert +.Horizontal Alignment +ĠQ ed +ãģĦ ãģ¾ãģĻ +/g it +ĠY YYY +ĠCard iff +Ġap a +organ ic +ĠWhere as +Ġæ Ŀ +ĠM ia +Ġdemol ition +Ġsc ars +Ġp ai +Ġre tries +Ġr q +ĠDen is +( Utils +Ġallev iate +ĠP IC +id ue +Ġacknowled ging +Ġ// //////////////////////////////// +ç¡® å®ļ +Ä « +\ Json +.b inary +Ġx type +sign als +ĠAp pearance +& r +} s +C i +ĠI llum +por ate +h og +Ġindex Of +\ Command +_par allel +ĠSher lock +í ĥ +Ġ" ")čĊ +//////////////////////////////////////////////////////////////// //////////////////////////////// +Ġcritic ize +ĠSo ap +ĠMatch er +Ġgr illed +* T +Ġad ore +ull ing +Ġjed och +_ref s +lean up +ĠJ AXB +Ġro ses +ĠL iam +size i +Ġget char +Ġtar de +-to oltip +Ġqual ifier +ĠInter mediate +_W indow +ĠMal ta +Dis connect +ew here +Camp o +Ġirr ational +led o +ĠD N +ARG V +Ġout ro +Ġth irteen +Jose ph +M AR +/g l +J ess +ĠPsych iat +Ġpadding Bottom +- loop +/ fonts +_se en +Te ams +React DOM +(m an +(x path +.get SimpleName +>( * +ĠP vt +Ġel ders +Ġp ies +.user Agent +- region +ĠGree ks +(f ragment +st u +Ġcouncil s +Ġst amina +ĠGod dess +è ¥¿ +Ġphilosoph ers +Ġpers one +ĠL ose +ĠCL R +ĠD ocs +Ġso ak +ĠHOLD ER +Ġb ells +hash Code +R ATE +_WE IGHT +in ous +end ra +oph obic +Ġpro se +Ġfin ely +/o auth +(s pace +ad ge +ĠM ama +Ġstring Buffer +Ġst int +Ġmis ma +Ġvill ains +ĠCrime a +Ġdipl oma +Ġпо Ñģл +ĠBe a +(j oin +Ġíķ ´ +CH AT +per ing +ĠC ros +Ġmon keys +Ġpred s +yl a +,, , +Ġvibr ator +ĠN U +åħ Ī +f ant +z et +Ġb ietet +un ft +sw orth +.F low +Ġpsy ched +ĠContin ental +> t +Ġqu ilt +. UP +Ġexpans ive +Dis pose +(l anguage +C aps +_Z ONE +Ġrec ycle +ĠMan aged +current Color +.b roadcast +sign In +.p rom +ll u +ue blo +Ġpunch es +Ġautom at +Ġassign ing +Ġcreate User +ĠAll ied +Ġconduct or +Ĥ ¨ +Ġs addle +Ġd ni +omed ical +-W est +Positive Button +Ġit alic +? [ +(tr igger +Ġele phants +":" "," +Ġcal iber +raft ed +d igits +Ġmar shal +mill iseconds +mark ers +m om +/ place +Ġhol istic +: t +# , +Ġb oto +Ġnause a +ĠSh ooting +ite ch +Ġtext Status +< Class +ĠDes cribe +Ġbuff et +g il +Ġlog its +std call +mod s +ĠSk ull +ĠB are +h ope +ĠIn tr +F air +ĉ pt +Ġacompan h +Ġf kk +_r pc +Inst alled +_ ans +.get Minutes +âĢ¦ "ĊĊ +- thread +Ġpres chool +AIL S +Ġdiff ic +( convert +ĠN ath +ĠDO J +Ġreg imes +Ġenthusi ast +Ġwarrant ies +Ġfasc inated +_b inding +_N ot +oft en +_R W +/m ail +Ġtitle Label +Ġvill agers +ĠJ iang +Ġsw agger +.Row Index +_img s +rap y +VER AGE +. Up +Ġno op +c io +ĉ ST +Ġdecre ment +Ġmagn esium +_ rotate +S it +Ġnieu we +Ġter med +íķ ©ëĭĪëĭ¤ +Ġur g +_t ouch +Ġsw arm +Ġcl ave +th est +ĠL af +H X +ĠH ulk +Ġplaint ext +ĠSof a +get Session +L ed +Ġecosystem s +he i +ĠK ills +Ġhus bands +Ñħ ÑĢан +(d om +_t iles +Nib Name +Ġdon ating +. acc +Ġlifes pan +.b n +_RG CTX +æ ¥ +ans en +Ġmod elling +Layout Params +ĠonChange Text +rs a +- location +.P e +(b us +(s ong +Ġprodu k +ĠSH OULD +ĠC J +Ġs os +ĠHome Controller +.load ed +(D ocument +.s ocial +t iles +Ġl ame += df +.parse Long +Ġpr ac +Ġdet ox +ĠV E +Ġpunt os +Ġdo ctr +Ġan cor +CA PE +Ġc mb +çĦ ¶ +*) " +:// / +Value Type +Ġmort gages +; q +ĠRock ets +s port +UG C +ct s +ãĤ ģ +ie ur +ĠAppe al +(n b +//////////////////////////////////////////////// //////// +IM ATION +ĠC res +ĠMan ip +C ause +at ypes +man ufacturer +# ---------------------------------------------------------------------------- +Ġsp or +es on +Ġpun ched +Ġbook marks +ĠBul k +Complete Listener +ĠTalk ing +ĠEr nest +Ġrub bish +k ills +ĠDE FIN +Ġneighbour ing +ar lo +ĠP CA +ĉm atrix +lo k +Ġat las +ĠG ur +Ġw yn +-n egative +Ġt ul +Ġre lic +ĠV oltage +ĠPre is +ĠJ NICALL +ĠPM ID +ak et +ĉ attr +Ġet iqu +ĠM J +ĠG mail +cl r +_exec ution +éĶ ® +pos itor +. af +N r +Ge orgia +Top ology +Ġperch é +Ġmus lim +Ġepid emi +Ġsab ot +act us +Ġë ĮĢ +ĠIO Error +. est +p refs +ĠKr ish +.Read Key +NAS A +u ção +_D b +umer ator +W ide +(st atement +.end point +.... ..... +Ġ[ * +stream s +m time +P x +at r +Ġt pl +R oman +Ġscen ic +.n z +ĠSe conds +sub menu +Ġìĭ ¤í +_b undle +Ġde ÄŁ +ĠS isters +pre ferences +Ġport a +Ad visor +max Length +ĠG REAT +__ (Ċ +ole st +ĠLabel s +Ġen fer +ĠĠĠĠĠĠ ĊĊ +ĠThe ft +_F ILL +ĠW ise +) application +un ami +> ())Ċ +ADD RESS +B ST +et zt +ĠQ gs +S ense +Exception Handler +ĠCh u +.get OwnProperty +Ġexerc ised +iot ic +ĠRe leases +Ġp interest +ol ie +is oft +Ġsequ encing +Ġpad re +] ));čĊ +(r adius +.m ed +aint ies +.Object Model +Ġem ple +Ġseg uro +St ars +Ġqual itative +lem n +á» ± +> "). +Ġg x +-c ert +ĠAST M +Ġfull name +Ġte lemetry +ĠCamb odia +_ ul +ĠCl are +C USTOM +Q C +ĠUn s +ĠHTTP S +ĠPark inson +ancy box +',' . +T ue +.get Last +Ġab i +Äħ d +A st +ĠEd iting +.Un ity +j mp +Ġm ats +Ġshared Preferences +Capt ain +.page Size +Ġr tl +Ġan meld +Runtime Object +Ġdemand e +(" ; +se ite +-head ed +ĠK ra +ĠF ONT +` \ +Class NotFoundException +. avg +atic al +A j +Ġpermit ting +Pro j +ERR Q +Ġcre ampie +ĠBuy er +-mod ules +ĠSund ays +| `Ċ +Ġday time +Ġ+ ( +Ġgl itch +ĠOper and +Ġtox ins +iny a +D NS +ĠS as +C ake +ĠNation als +.add To +Ġs inking +Ġcompreh ension +Ġsc or +ag ements +Ġt ard +Ġmarch ing +ĠM TV +Ġs ane +Create Info +Ạ¯ +Ġend Index +ĉ layout +ĠåIJ į +S ITE +ĠT HERE +Ġ[ {' +opath ic +Ġtrans mitter +/ body +Ġp und +ĠC losing +Ġset attr +Ġbound ed +At las +sum ing +(t imes +par er +yn om +fe it +Ġf rem +- leg +ĠBr as +> # +Ġì¶ ľëł¥ +ĠIN STANCE +ĠC ouch +_host s +lik elihood +.M arker +ĠM asks +Ġcere al +util ities +Ġelement al +Ġdist orted +in active +c ry +W L +UPPORT ED +.Th rows +/s chema +ser ie +." ', +ĠBened ict +-p icker +ig gs +ĠPir ate +åij¨ æľŁ +ĠTh ema +ĠSouth ampton +Ġarray With +ĠPaul a +Ġpredict or +- Ass +.user id +Ġper i +Ġexagger ated +ur ate +arse ille +ĠCon cent +ĠP ik +Ġ@ _;ĊĊ +Ġform ations +Ġden omin +"/> .Ċ +ended or +Ġpan cre +Ġam t +Ġon Resume +on Delete +ĠB CH +) (" +m ovement +Ġpot assium + čĊčĊ +ĠMah m +} ";ĊĊ +Ġd q +ĠPublish ers +ĠAm pl +ĠDani elle +Ġt ern +èµ · +no ÅĽÄĩ +e in +ĠAsync Storage +un ger +rou w +Ġsc issors +/ assert +.b ucket +/ archive +_M an +Ġint oler +Ġ() => +ĠÐĴ Ñĭ +Ġsa i +.x y +." čĊ +Ġur inary +es ub +IST ICS +ĠÎ º +Ġcompl iments +Ġtypings Japgolly +ih ar +Exp ansion +ĠS erving +_st udents +ĠX BOOLE +( il +Ġì² ĺ +Ġj ó +(t ol +( JS +ĉC G +ĠD RAW +tw ig +Ġo at +_sm ooth +ĠC SL +Ġos ob +Ġens uing +Ġbank er +ĠBack pack +_p ing +Ġwish list += ax +ĉĠĠĠ Ċ +Dis ney +stead y +"> % +Ġproph ets +ĠZ X +Ġminimal ist +.PL AIN +Se attle +. ordinal +ĠPI PE +Ġret orna +Ġjug ador +ĠB ret +ĠâĶ ľ +Ġpl ush +UL ATOR +Sort ing +.grid y +ect omy +_ activ +r ack +Inter active +ĠAntar ctica +Ġv engeance +en so +_k nown +up plier +.Mod ules +ĠConnection State +éļ IJèĹı +@ FindBy +Ġpl acer +\ model +< ()> +.is Successful +-g ood +b z +ĠDr aco +Ass istant +-ex tra +аб лиÑĨ +Ġhyp ocrisy +Ġt st +ĠA gr +$ txt +Ġlog istic +lic ensed +ĠH of +Ġt at +( iv +Ġinto xic +post Id +_st rike +Ġhum iliation +pc odes +" sync +(rec ipe ++ N +rent e +ĉ Client +ycop g +ĠZur ich +ĠPro files +C ountries +Ġp ict +Ġroll out +requ encies +Ġpatch ed +Ġcar tridges +Ġsh ading +J ar +Ġsalv age +ĠTax es +Ġstand by +apor an +E igen +. angular +ĠN ested +äº « +Ġis Visible +ĠDw ight +_BR ANCH +.D elay +Ġk end +Ġfacilit ated +.flat Map +Ġs anta +ĉS end +/m essages +Ġof Type +ĉs wap +# plt +ĠTur ks +N ES +Ġprogress ively +ĠRes idence +ĠT REE +Ġno en +d io +Ġn elle +Ġsog ar +itt i +week ly +Ġambigu ity +_Set tings +W are +.ne o +_D ST +Ġæĸ ¹ +pre p +lob by +@ email +/m ovie +Ġfun kc +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ÂŃ s +Ġguard ians +- pos +Ġconfig uring +ĠC PS +ĠDe us +Ġvidé os +_ empresa +Ġsl apped +< Model +Ġunders cores +U h +.access Token +SET S +ĠS parse +ĠCal d +: path +ĠS ervers += batch +Ġkn itting +Ġx a +Ġsearch Bar +Ġsn ag +Ġinf used +.b am +le ver +Ġtax onomy +Ã İ +Ġatt aching +Ġh ern +_N OP +Click able +(P arse +ĠDynam o +-b uilder +Ġdere g +Ġsc attering +è¿Ľ è¡Į +an zi +ĠShe pard +"> ',Ċ +_X DECREF +ĠBuzz Feed +_M ARGIN +P LOY +.sm all +Ġm imeType +Ġh olog +ĉc amera +li as +Ġsusp ense +ody nam +b au +Ġgrave yard +_n amed +":" ' +Ġ******************************** **************** +Ġgame Over +ĠLENG TH +ĉs creen +Ġdo InBackground +_depend encies +Ġr tc +/ up +_ ROM +H all +Ġdef iciencies +( te +' # +_e quiv +Ġpre order +ĠA xe +ом Ñĥ +.send File +Ġfil t +ĠLim its +ĠCaval iers +.dis count +âĨ IJ +ĠW it +QRST UV +Ġi j +Ġt egen +Ġ: ", +diff iculty +p unkt +ĠEmail s +ch lor +(f un +.U int +ĠSt all +_ verified +u D +File Type +Ġple asures +Ġjud iciary +Ġsh am +ip ur +_PL US +off ers +( foo +_G T +ĉc ore +ENT ION +ĠLib eration +Command Line +_de partment +.A r +_ne ighbor +ĠSub mitted +ĠĊ +Ġdro its +Ġhomosexual s +Ġab duction +ĉw idget +$ headers +ĠD AR +Ġfl a +th reat +Ġlou is +.Get Property +" Just +(f rames +ry o +prof ession +| i +íķ´ ìĦľ +(s v +Ġun recognized +I onic +F ashion +Screen State +ĠIn coming +Not Nil +Ġsync ing +em ie +Ġtherm o +_pro cs +Ġincons istency +rel igious +.m j +Ġperson n +Ġmoment os +or arily +Ġæ Ĭ +_ne urons +Ill ustr +im oto +il ik +ĠW oj +Tr ading +Ġapp are +Ġentre prises +ach at +Ġ ¬ +Ġne igh +BUTTON DOWN +ĠMah er +ag han +-h ash +" f +Ġclient ele +.add Button +ĉ SP +Q i +Ġgr ated +POS ITE +: > +ĠHow ell +ĠCompar ative +ĠIS C +ÂŃ i +O cean +D avis +ĠFil me +W ins +ĠJ IT +oc cer +ĠC orm +ENCH MARK +rch ive +ica ção +Ġm ata +Ġchild birth +ĠOption ally +En s +Ġx http +Ġel ucid +_Osc InitStruct +)) ):Ċ +Ġint uit +ĠDon ate +Ġcorrel ates +> Delete +Ġequ ipe +Ġb oca +Ġinfl atable +er ah +ĠDateTime Kind +Ġcal ves +\ Lib +Ġem lrt +ĠTr ilogy +ĠP anc +ĠD uis +ĠpelÃŃcul a +WAR DS +_DE TECT +-section al +dh cp +For Row +-de struct +ĠPres enter +/s lick +, on +ĠCit adel +logged in +_sub type +Ġsig ue +Ġc uring +ĠFire wall +Ġfluores cence +ĠItal ians +иÑĤ ÑģÑı +.get Style +In Seconds +j ie +-S mith +Ġx link +Ġsub missive +он ÑĤ +arbon ate +ĠF aul +_go als +ĠCommission ers +chart Instance +_POST FIELDS +Ġmed ial +Ġman os +Ġdel t +sv m +.Ap is +ep hy +Ġasym pt +Ġapp Delegate +Ġimpro bable +ck a +sim d +/ Error +. âĢĵ +ĠP TS +de er +Ġs ina +m agnitude +ID ADE +'] }' +Ġmay ores +ĉ comment +/ console +" @ +v olt +.s ell +ĠM acy +Ġmel od +Ġim ágenes +_ch g +Ġin out +ident e +) '),Ċ +d ni +.b lob +Ġtyp ography +Ġe erie +_O ID +pes an +aj an +Ġch opping +Ġbl uff +ad f +_b ases +.Form atter +Ġ\ % +ĠPage Info +Car rier +ĠCal ibration +com o +-b odied +Ġfinanc ier +ĠIN A +. ERR +Ġhood ie +ĠSan ity +gu arded +.opend aylight +ISM ATCH +High lights +ün k +ani em +anger ed +assign ments +Ġregistr ado +ĠU PPER +ampil kan +ash ire +ĠNik ola +ĠC FL +ĠH DC +Ġp oids +ĠIP s +Ġprevent ative +ips oid +if ix +.c amel +.g a +V olumes +- ste +Y ahoo +_s ibling +H ighest +opt group +Ġkvin na +âĢĿ ãĢĤĊĊ +ĠAppl iances +Ġ" >< +') ")Ċ +ht t +ĠIdent ified +Ġpenc ils +Ġmember Id +Ġappend String +.load Data +Ġmock Mvc +Ġj ub +ĠSl ut +ĠTai pei +st att +Pol it +Ġpart ager +Did Change +Incre ases +) }. +ĠB aba +_CL IP +[ unit +Ġк лÑİÑĩ +Ġalc uni +ĠL ola +Ġcl inging +@ PostMapping +(con cat +Ġss id +ĠFa uc +ok it +ĠRecord ed +á lez +($ ('< +.assertIs Not +Ġk ali +V olt +Ġwarm ly +Ġsca res +get ti +füh rt +_d oes +. EMAIL +im ations +Ġspring fox +ĠDec om +arc y +Ġgl itches +ĠM off +ĠV oll +.b etween +Ġcoord en +ĠPart icularly +GB P +Ġsem ble +East ern +_M SB +]) {čĊ +m organ +ĠE VAL +d ere +HO USE +mo ire +ist ique +_l stm +-com mit +yster ious +Ġtw ink +-th umbnails +en ÃŃ +:' ', +Ġblack out +ĠFlo ors +Ġso fas +Ġou i +lesh oot +ĠRa q +- abs +Ġk ra +M ining +sha ft +.set Columns +Cl azz +PRE TTY +.play list +éĸ ¢ +-Sah aran +M ING +ĉ bl +è® ® +j f +DO CKER +hope fully +( ignore +ĠUsers Controller +ĠMitar beiter +ĠL ES +Ham ilton +-m etadata +ĠK K +ikt ig +Ġwoll te +egr ator +] bool +, current +Ġvalue Type +Ġexcav ation +ol and +Ġv erv +/file path +Auth Provider +Ġpro crast +ĉ ULONG +_MEM BERS +Ġup lift +ĠAut onomous +Ġart works +ĠOut reach +Ġp ore +Home page +Dialog Title +ĠGener ating +PAR SE +Ġsem anas +Ġhuman o +JSGlobal Scope +Ġvol te +Ġb ella +(is instance +Ġpl c +\C atalog +Ġeste emed +éĽ · +(s uffix +Ġswe eps +ĉ ORDER +Ġdo ivent +ĠSw arm +ĠComp iled +get Page +AD R +.R ichTextBox +ĠN aming +ag ged +ĠG ANG +r asing +ode led +Ġg ala +ĠJS Name +dd f +Ġill ust +ĠLans ing +[ port +-de ath +Ġdin heiro +ĠE ighth +Ġb ian +st Ã¥ +Ġvers ión +ĠLinear Gradient +ĠHard ing +. *) +ec zy +$ header +Ġv Ã¥r +Un checked +Ġko je +ĠPal adin +() )), +G iving +() })Ċ +Ġd ips +F riendly +Ġport rays +Ġhel ium +Ġinsurg ency +_ex piry +ĠstringByAppending String +Ġa antal +s lope +m ast +.get Integer +Ġ################ ######## +_PIPE LINE +Ġdens ely +Ġmut ating +m idi +ĠSe it +ay ne +NOW LED +ĠDes mond +ĠF Name +ĠN airobi +\ Context +Ġcalc ular +-d en +Ġc ott +] ):čĊ +ĠRecommend ation +ĠRole x +Ġvalidation Result +.p at +Ġn Ãły +ĠRest Client +ĠG PI +ĠAshe ville +ĠO SP +ĠPER MISSION +ÐĶ аÑĤа +/ notification +K night +_W ord +ĠB ender +rank ing +Ġpart ida +_res ervation +Ì Ģ +Ġm Name +Ġget ch +Ġb orr +Ġdilig ent +Disc uss +æŃ£ åľ¨ +ape ake +ion ed +-N azi +.c um +ĠK ron +=$ ('# +/s ingle +Ġerot isch +ĠV ib +Ġrat ified +Ġconcert ed +ĠREG ARD +Ġdo br +.Driver Manager +' r +Port able +ĉs uite +Ġrel aciones +ĠD op +emplo i +DO B +Ġcr umbs +Ġx ls +_App lication +(': ', +Ġ---------------------------------------------------------------- --------Ċ +m se +Ġber k +ĠReturn Value +ĠBel ly +Ġcam ar +ĠPe ek +els ing +Ġnot ifies +ĠTr istan +ĠG AR +em me +ĠElev ated +_C SV +(ch alk +Ġtw enties +ĠSearch Result += search +ĠMix ing +ý t +Ġrecru iter +ĠIDE OGRAPH +ĠA go +( Operation +$ values +Ġworld ly +ĠRosen berg +ĠConfigure Services +>* Ċ +Ġsn ork +_op acity +ĠinitWith NibName +i ado +A AC +Ġ] ). +; z +_par agraph +Ġnos es +stand s +if r +_m E +I raq +.P redicate +ena ire +]] ];Ċ +Ġun idad +Ġretire es +_h ello +Ġmode le +ĠUIT ableViewController +f write +_num ero +_vis ited +Ġrece be +( Notification +Fant astic +_sub menu +ĠP EM +ĠCup ertino +approx imately +class ed +.Read String +Ġdomic ile +_P W +Ġball park +ĠK ale +con tra +_f avorite +/ of +Qu ite +ĠOT A +Ġacceler ometer +did n +| ^ +ĠRohing ya +ivic rm +ann abin +обÑĭ ÑĤи +or ado +') + +Ha unted +, ID +( UIAlertAction +ur v +_b el +ĠMex icans +/ terms +ĠPaint er +Input Label +ĠV inci +ĠRos ie +\ uc +< Menu +Ġcool ant +(current User +_d ual +) "},Ċ +& p +Ġconver ged +Ġrestr ain +ĠYugosl avia += target +Ġimp uls +ds a +Search Tree +Ġh box +ĠImp ress +§ Ãĥ +get FullYear +(d a +ĠY YS +.al ignment +.Get Text +.token ize +ĠOlymp us +Ġmur ky +ore station +Ġdiss atisfaction +ĉT Array +_ kses +.Add Singleton +ĠStart Time +Ġfan atic +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +Ġentity Type +. override +Ġ ------------- +ĠDat agram +f out +(with Id +Ġ# __ +Ł èĥ½ +ek yll +.f riends +ame leon +Ġz ach +.simple Button +ret orno +Ġkon k +/s mall +ĠQuick ly +un read +Don ate +Detail View +Ġdu a +Ġpenetr ated +OM UX +Ġn ir +_p data +"], [" +Ġlow es +Ġdop ing +Ġas ymmetric +Ġneed less +our cem +Ġup ro +ĠGu zzle +af b +Ġsext reffen +-c ollar +Ġcol ossal +Mon key +n ish +Ġhandle Message +Incre ased +* dx +ĠChatt anooga +f org +ĠOr den +Ġsh ri +ĠV and +Ġ" @" +Image Sharp +ĠWild cats +pon ible +.sc enes +Ġpaint ers +ĠPf izer +ĠZ ah +To Local +ĠFl am +Ġé taient +)) ^ +ĠSand box +ĠTR ADE +Ġchrom ium +Ġac claim +Ġpac man +´ t +) reader +M ari +.Dispatch er +.A DMIN +ĠRem ed +Sw eden +Ġoverl ays +. er +Ġp ang +Ġclean ly +aven port +Toy ota +patch es +Ġv tx +ĠE is +cl ado +ĠR itch +RO LS +Ġh ade +Ġconspic uous +Ġdo cks +(j q +ĠPrem iership +ĠBe z +ĠâĦ ĸ +ĠÑĥ Ñģл +_tot als +Ġprov a +ĠC ue +Ġsa úde +ĠGame Controller +IM IZE +, port +ãĢĤ ( +.C decl +Instant iationException +Ġcoll age +ĠIO C +Ġb ais +Ġon Finish +-st ars +set Size +Ġmog ul +Ġdis illusion +Ġche vy +(S chedulers +( IR +_loc s +Ġcann ons +Ġcancell ing +/b us +Ġbuf io +ĠY ours +ĠPik achu +Ġter me +r Ã¥ +f ahren +Ġowner Id +Ġoblig atory +Ġcul p +Ġacid ity +-m ult +ĠBam boo +Ġ' "> +_g s +Ġcomp il +n ard +-ex c +Ġrh yme +Ġbut to +s ays +ant asy +ë ¸ +Ġcitt Ãł +Ġche g +Time String +Ġpos itivity +ĠD abei +Ġw ang +Ġes cre +" c +ĉv ideo +ĠRank ed +.str ings +>> >( +Ġин ÑĤеÑĢ +Ġrest a +[: ,: +Ġrend re +Ġdes er +J os +Ġdis ruptions +Ġоп еÑĢ +s ampling +sup press +Ġcontainer View +ĠSeam less +Ġair y +Ġon load +.Window Manager +ĠPL A +br aco +.set PositiveButton +Ġp du +Ġg si +ĠC li +_gr adients +Ñı д +ĠWh isper +c stdint +Ġl äng +Ġform ulations +én om +ourn emouth +[$ _ +Ġordin arily +.set Username +Ġfacult ies +MIT TED +/ values +Ġwe ir +ĠA pt +M Z +ĉc f +uck en +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉĉĉĉĉ +def ense +[i Var +ĠBusiness Exception +Select ors +(co ordinates +ĠRes ets +ĠDr inks +ole ans +(st ypy +_IO C +.x xx +ĠSl ater +ĠBel ize +Ġ/ ************************************************************************ +add in +_ep isodes +Ġis chem +legal ArgumentException +D anny +Ġp ared +.code haus +ĠAss y +ĉ Rect +â ŀ +.list a +Ġв аÑĪ +Ġv ets +HW ND +ison er +Ġx o +Ġor ally +ĠSt mt +.r nn +ĠD PI +ĠStr ikes +.setViewport View +Ġèĩª åĬ¨çĶŁæĪIJ +Y ELLOW +GL enum +part ners +ĠImp licit +Ġtak o +âĢĻ elle +Ġerm ög +total Count +G il +ĉ work +Ġpr atic +in ati +ab ies +ĠSk inner +Ġspir ited +Ġpancre atic +Ġh df +' em +Ġpsych osis +olic it +Ġ" {" +_at ual +Ġé lect +TE AM +Ġd ak +ĠSW AT +.Fragment Manager +Ġprovision ing +l ifetime +_EXTENSION S +ĠC ASCADE +Ġ! [ +(K P +Ġv em +ĠInterr acial +'] },Ċ +sp acer +_k v +W arehouse +R DD +_f sm +.Stretch Image +, Yes +ĠRefuge e +ĠBr inging +Ġv álido +.inter section +Ġsp ooky +_port al +Ġmo th +ĠZ odiac +ĠSOC IAL +M imeType +'] }} +_Bl ue +Ġbot anical +Ġfr ags +Ġfamil ial +- du +Ġse izing +(block s +.r d +.check NotNull +Ġmis er +Ġmax x +ĠK nee +View Item +Inner HTML +D anger +(( __ +Ġprz ypad +create Url +** , +ĠDecor ating +ATEG Y +?> / +.Design er +hex digest +ĠEvery where +all eries +.TEXT URE +.Block s +z ell +Ġpre ço +S uddenly +input Email +(s ync +.b d +gold en +> '); +ĠDick inson +>> (Ċ +ĠQUE UE +Ġget Column +ĠS AND +.p iece +lic er +Fl utter +Ġget Version +Ġresource Id +og l +ÅĤ aw +.Br anch +ĉ web +Ġfr amerate +PP P +Ġfr ay +C NT +Ġinformat ie +'] čĊčĊ +ne as +Header Code +Ġæ ¸ +Ġtr g +raw types +H onda +Ġmark eter +Ġrequest Data +ĠP g +ĉ not +Ġpage Info +Ġakt uellen +ãģķ ãĤĵ +ĠA MS +push ViewController +ĉ AL +Ġv ests +produ ce +-m ême +ĠRah man +F unny +E Z +_ Valid +Ġsquad ron +Ġl ash +Ġ irm +ias co +ĠPar an +Ġpet ites +ĠDec ay +Ġun initialized +priv ileged +Ġm bedtls +å¤ĩ 注 +Ġ^ . +Ġec static +D etroit +Ġpart en +Ġsou venir +.get Login +моÑĤ ÑĢ +en ção +ĠmÃŃn imo +ĠAccess ed +ri ó +M ic +ĠV ocal +.Set String +Ġmens ajes +åĢ į +Ġattr avers +ĠA ph +Ġ' );čĊ +ünd e +Ġench anted +ĠRoot State +ĠCLOSE D +ĉĉĉĉĉĉĉĉ čĊ +Ġcal iente +or ris +Ġphysic ists +h wnd +_v i +Ġráp ido +Ġcapital ized +ed By +Ġmach ining +Ġhub by +ĠSt acy +.B us +dr ink +H ur +Ġprop ia +Unit Test +Ġmiscon ception +__ ));Ċ +/d c +ĠMay weather +_m C +.create From +ĠQ Painter +rops ych +inn itus +ay as +Ġg eg +(d w +Ġus ado +Ġtrick le +Ġann ihil +ĠP asta +Ġ++ Ċ +(Expected Conditions +.post Value +ic ap +ĠDon etsk +_s oup +-p ublish +ĠP b +ment ions +AC CEPT +.P ull +,âĢĻ âĢĻ +Ġret arded +_AT OM +ĠTermin ator +-c ourt +ĠCLLocation Coordinate +Ġrever ence +ĠS SC +ut ely +ĠW ON +ĠG SL +fre i +.get Longitude +Ġopen FileDialog +.B utter +- important +_M ANY +ĠG ong +âĢľ How +Ġg orge += msg +ĠEz ek +create Command +: checked +Ġinf ographic +.W EST +Dir s +Ġguard a +Ġbeet le +< small +- android +Ġcred itor +ĠM éd +Ġfinal ist +Ġab l +ne v +_inter action +ĠMonter ey +j ah +Ġcand ies +ĠQu incy +èª Ń +Ġbatch Size +ak it +Ġo be +(p ara +Ġexperiment ed +Ġcouncill ors +Ġcl ashed +s qu +-st rokes +ĠG K +ĠEx pires +Ġprosec utions +ĠCreat ures +Ġy ö +x lim +_IM P +Entry Point +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +.Default CellStyle +Ġbre ve +ĠBrit ann +Ġsweat y +Ġle th +Ġflash back +per manent +ĠJ DK +_D etails +E uro +p pt +Ġrich TextBox +/ board +Ġtr ance +.c ycle +'); ");Ċ +Ġtox in +_de init +Ġover arching +Ġconfig parser +ĠKaw asaki +.th umb +Ġplay a +ĠJose f ++ _ +Ġzero es +Ġa up +ĠH ari +comm itted +N it +.file Path +ĠDis abilities +man ufact +-al igned +.RE SET +Ġrust y +E y +Ġou sted +cos a +Struct ured +.get D +Ġs ábado +> Loading +_m A +.get Random +bl ings +Ġchees es +tt i +. âĢ¢ +ĠBurg ess +ender it +. ',čĊ +(" "+ +ac b +% p +index ed +_pred icate +nes ia +Ġb ied +ĠC IT +( Pos +_r adi +ä»· æł¼ +B iz +ĠAdoles cent +Ġvi ên +c ycl +_C ancel +Ġcon clusive +Ġappell ate +inform atics +S J +Ġelect ive +role Id +Fetch er +ĉ Command +(" (% +Ġf art +IL A +get Block +A USE +Ġд ан +ĠAr te +Ġnot ifying +Ġge le +.s ame +ĠReg el +ĠBa ÅŁ +.c reation +ĠV N +_comm unity +Ġuns ustainable +SE X +Ġgrid Size +res cia +avers able +(', ')[ +ĠPh elps +á»ķ i +ANCE LED +- IS +.run ners +ĠSt okes +.P rodu +Ġwh ipping +_ac quire +Ġinvestig ación +f ried +.copy With +ĠHard cover +- Se +áŀ¶ áŀ +inv itation +les ai +ĠD orm +ĠÑģпиÑģ ка +Ġconcaten ated +oph il +Ġthink er +/font awesome +ĠLe opard +Ġ"/ ");Ċ +Ġresidual s +ĠMic rowave +Ġconform e +th rop +Ġdis emb +ĠO MG +ĠDisc ipline +ĠAc robat +/re pository +df a +_M ED +buf io +Ġméth ode +_H OLD +ias i +_ legacy +) ččĊ +æ£ Ģ +Get ProcAddress +Ġy ay +ot ence +order id +-t w +Ġdear ly +In coming +/ il +Ġneu rop +uc z +); čččĊ +ĠInnov ative +Ġprof und +ig mat +Selection Mode +re levant +.G O +Ġbru ises +Ġs ach +ode f +Ġre imb +/d esktop +-s pot +und ance +Ent ropy +\ core +Ġsug er +ĠM vc +ĠGN OME +_ind x +ĠYY STYPE +ĠMat lab +ĠC IF +Ġ* )) +Ġproduct List +ĠAl right +ac emark +ÑĤи в +mod ification +int ernational +Ġhom ers +Ġdict s +ĠQ Font +.SQL ite +Ġtransplant ation +ĠMessageBox Button +ĠEl ves +'] ])Ċ +(Q Icon +Ġcin emas +CO ORD +- China +Ġkh ẩu +æĪij çļĦ +Ġskull s +Ġpain staking +f ce +.XR Label +Ġspec ifier +Ġpref erring +/ activity +( Photo +á lt +.l ot +' '. +ann once +.google code +-p df +ĠP oke +_A CL +Ġend owed +dis cover +.om g +Ġwood land +.M agic +Ġvol ont +Not Allowed +Ġch ave +BM W +',' =', +ĠS IX +æĪij 们 +Ġkos her +Ġaspir ation +int l +_ref ptr +'+ Ċ +ment or +.cl ub +Window State +.A RR +Ġz za +Ġmessage Type +.e qu +Th or +Ġin just +Ġg ums +Ġborder Side +//// / +ĠTrans mit +Ġbuf size +Ġh ak +Ġell as +R ANDOM +ĉm c +Ġpe a +ek o +document o +Ġhyster ia +Ġaren as +Ġgun men +Ġm ike +Ġimp unity +atis ation +_Z ero +_COMP ANY +ĠG ors +Ġuse Class +( redis +ĠRUN NING +ĠB air +vel te +Ġ',' . +аÑĤÑĮ ÑģÑı +ö st +encode URIComponent +_re strict +Ġdec als +ĠPed ido +Ġalter cation +Dis plays +ĠApp licants +C US +Text area +ĠAng ola +.f uture +ĠUS HORT +Ġsuppress ing +Ġset zen +AP olynomial +Ġto ch +Ġhall mark +Ġ$ $$ +ĠCHAR SET +.r pm +ĠD ich +---------------- ---- +_p arm +è¿ ĺ +acc iones +h ait +WAR DED +_r outing +ĠN OM +Ġen clave +ĠLot to +ĉf r +complex Content +ĠBall ard +k ube +/w in +.getColumn Model +_RE PLACE +Header Value +Ġest udiantes +Ġap is +Ġb pm +ĠType Name +And Get +rit a +Pl ans +> Note +Ġfet isch +Ġton ed +_g oto +ons ense +Ġm olds +Ġinfiltr ation +ĠGuerr ero +ub bo +ck i +($ (". +_ activities +(ch anges +Ġof App +ĠKe pler +ĠD emp +ĠCont inent +.T icks +ĠUn signed +ĠJah res +Ġfresh men +ĠArch ived +ĠкоÑĤоÑĢ Ñĭй +Ġ' :: +T utorial +C c +Ġtable LayoutPanel +from Json +.level s +_trans ient +Ġendors ing +ĠD IC +la uf +Ġsh red +_E MIT +ific antly +AL A +/ proto +Ġnarrow ing +U tc +Fact ors +Ġsent ient +æŀ IJ +lix ir +ĠC ROSS +met eor +Ġgro in +Ġm db +ĠRot terdam +Ġcom ida +ĠOp Code +ĠDefault Value +Permissions Result +Ġheter ogeneous +Ġm oot +Ġde ceived +-in dependent +ĠObject OutputStream +Ġover power +.d up +Ġl db +Ġdomest ically +Ġbest ellen +Ġlo v +ĠContract ors +Tri angles +Ġfod der +Ġfilm es +ä¼ ģ +Ġrev olver +Startup Script +/ validation +ĠResource Type +i ÅŁ +ĠL az +f ef +Ġlst m +{ * +. attachment +.h its +ew ith +DO G +Al abama +Ġmedium s +.m Context +-c ols +åı ĭ +.not ice +Ġat tn +ĠP acking +ĠL n +_COM PLEX +/ Users +.sav etxt +ĠR ounds +?,?, ?,?, +Ġing l +ĠR OC +_f emale +ĠSt ard +]] ; +Ġwrest lers +Ġtorrent s +Ġsin h + ĊĊ +ë³ µ +s ense +how ever +.Ph ysics +Inf rastructure +ĠSac r +F el +ĠD ISTRIBUT +é ments +ĠValid ates +################################################ ############ +Ġ| / +Ġes l +Ġré seau +ĠB ip +BY TES +_W ATER +Turn ing +EL S +Ġj uxtap +Ġlesb ische +ý ch +( Unknown +Ne o +@ JsonProperty +Ġal umnos +ĠRaq qa +ime i +.get Bounds +.Mouse EventHandler +#### ### +Generic Type +/c ms +Ġturn o +Ġм ин +Ġfolk lore +ĠE vo +Ġconduct ivity +Ġle ben +Ġgear box +-v s +ĠÏ Ĩ +Ġdrink ers +Ġcon exao +ĠTe eth +Ġget Arguments +ĠR AT +ent ious +E duc ++ W +ĠInstitution al +ĠB ord +is Equal +(p wd +Ġign ited +ĠR ousse +Ġimpact ful +ĠM alk +Ġg eral +ĠP ivot +Ġa zt +Ġcsv file +ĠR ope +ĠSOL UTION +ĠArbit rary +Ġlet to +.Mouse Adapter +Ġ} }} +ĠSail or +der a +Put ting +Ġconcentr ates +Ġauth Domain +âĢĿ çļĦ +-f inals +, strlen +Mu on +ĠOrd inary +fire fox +ĠLa TeX +ĠH und +engine ering +/ blue +ed TextBox +(" "); +ĠC DDL +ke pt +ĠGet String +K ir +() =' +ĠO CD +ant ium +$ menu +ĠAppalach ian +Secret ary +ë¥ ĺ +ี ย +Sem antic +Ġ* [ +est one +ung kin +Max Y +-t one +"} ;čĊ +_P art +< Member +tr am +Ġtrans istor +Ġ---------------------------------------------------------------- ----------Ċ +ĠDes de +Ġright ful +ĠCorn el +æ ij +.H OUR +Ġsidel ined +ref errer +m aze +Ġhol ster +Ġcripp led +ĠDate Formatter +oph age +_m D +Ġdes elect +ra ud +ĠPK K +row Data +Ġlock smith +.res ponses +(product Id +_ST MT +Key Type +.Th en +z ee +Ġcr t +ĠGrand ma +@ Resource +Ġbit wise +-c mpr +ãĢĤ www +zeit ig +& display +Cart Item +- No +Ġnum éro +Ġm aur +Ġinst ancia +ĉd t +_n pc +Ġskate board +âĢľ All +ĠCrow d +Ġä n +Ġb raz +ca e +yn et +/p m +/s creen +OPT ARG +ĠV Box +Ġle opard +_g reater +c pt +< dd +Ġmechan ically +osp els +) f +.l wjgl +.get Port +ĠP REF +.Add Transient +pp ard +Ġí ļĮ +Ether net +Ġsal ine +(level s +Ġservice Provider +.A ngle +alt itude +illa ume +Ġs cape +_CAL C +_ quest +ĠDiss ertation +ĠE DM +-C ds +Ġhon orary +st ops +Ġsub dir +ĠV H +ĠChe at +Ġright fully +Q E +.Write Byte +fig ures +enn ie +( DBG +Ġvoks ne +Ġexp ended +UN ICATION +il inx +ĠRec ap +_ verts +Ġtra umat +Ġget Player +Ġverb ess +Ġcultiv ating +Ġiniti ator +Th ông +find First +_per ms +Ġbu c +Ġ""" čĊčĊ +T YPES +object Manager +(Configuration Manager +Ġtim id +Ġsnap chat +Ġcon seg +ĉd istance +_right s +_D es +ĠF lesh +- ver +Ġa fl +fra uen +Ġblas ph +ĠQual ität +ma f +Monitor ing +.D iff +Ġshore line +Ġresponse Body +mem set +< decimal +Smarty HeaderCode +Ġin sets +ĠBinary Tree +amed a +Ġn ihil +ĠN ay +ym ology +ĠW G +Ġt api +ĠInst alled +m aintenance +)} "Ċ +ĠX O +-per iod +s ar +Ġning una +ORM AT +.set PrototypeOf +ĠK b +ĠHen rik +ét ique +ĠLah ore +ĉ Address +Ġmel ts +N y +_adv ance +Ġveloc idad +Ġalum no +Ġsanit izer +Ġph ishing +ĠCom et +Ġch iar +ĉs pec +trim med +(state arr +on nen +Re venue +L ens +Ġcha ired +ĠAss umes +Tr ash +_un set +\ Bridge +Point Size +ĠPol ic +Ġsex uales +ĉd fs +ĠWide String +Ġaccru ed +Y W +_S CHEDULE +Ġk ite +Ġparach ute +[ table +Ġactive ClassName +.Qu ad +Israel i +ĠÅ ĵ +Ġho og +Ġch á»ī +ew ear +Ġtire lessly +set Error +.get Amount +.set Items +ĠM anson +ĠBay esian +_F lag +AC HER +/ original +Ġimm ac +ĠLos ing +' >ĊĊ +L ic +ĠMir age +ĠAssembly FileVersion +Te V +ĠValue EventListener +-s olving +Th o +rou lette +_W P +Ġunint errupted +Ġfield Type +.T yped +Ġam our +Ġmock ery +(v ol +ĠSub committee +ĠR uf +ero x +:UIButtonType Custom +ĠBl ur +Ġwy kon +nc es +ASH BOARD +!! ");Ċ +Ġmurder ers +.d aily +ĠDI AG +j ing +Ġdol phin +Ġl òng +Ġb ö +ĠV ocabulary +.St Object +') "> +Ġz un +Ġscrim mage +tr éal +ĠL ig +[ vi +C ole +Ġfrost ing +.Pl ayers +- translate +Fe els +=\" / +.Butter Knife +Ġ?> ;Ċ +Ġav i +inn ie +.F ailure +Ġsp indle +Configuration Exception +_h op +Ġpos ição +ĠA wait +UIImage PickerController +ĉ day +Ġgen om +C ab +ĠÑĢ езÑĥлÑĮÑĤаÑĤ +OR IGINAL +Ġejac ulation +(t cp +SE COND +Ġton ic +ĠList Box +Ġ ĉĉĊ +() >Ċ +Ġqu atre +ượ ng +with Errors +.M aybe +, âĢ¦ +token Id +_UN DEF +Ġfresh ness +ĠAmend ments +.map box +.C V +(b log +_get time +. quest +s parse +Ġres ale +Ġenthusi astically +ĠProstit utas +W a +C argo +.Parcel able +SENS OR +ĠRy u +La ughs +_N ative +/ pg +yst s +Ġphot oc +ç® Ģ +ado pt +.spec ies +conc iliation +Adjust ed +.Firebase Auth +ut tle +ord ination +Ġm unch +ĠSt ake +.p ing +ank er +(QString Literal +Ġsub script +ĠĠ ĉĊ +ĠM CC +_C md +se xy +i ou +ĠM ANY +Ġn anny +TR AIN +Ġflour ishing +ĠW atches +ĠQ Map +ĠF erm +Ġwas m +ĠA bed +_ UD +ĠGlass es ++ v +Att end +.Ch ain +Ġdec ency +ĠSupplement ary +h unter +-t xt +Ġ" }";Ċ +.set WindowTitle +(" +Ġmasc ara +( Profile +åĬŁ èĥ½ +imit é +Ġwild fires +- ROM +.is On +(group Id +Re pair +accum ulate +Ġ< ", +Ġhand written +Ġach eter +ĠM GM +ĠIr ma +->{ _ +ge e +cr iminal +Ġèĭ¥ è¦ģ +Ġmoment arily +") != +_l it +Ġexpires In +." ). +éķ¿ 度 +Ġfr ække +vl c +Ġor bs +), $ +Ġvent ured +/ >\ +char m +N uitka +eld ig +aton in +W itness +-l at +Ġset Hidden +Ġrelic s +Ġcons ulate +. IGNORE +" After +Ġset Address +Ġbeste ht +Ġ'' )ĊĊ +.x axis +Ġser ão +Ġmis led +_UN IFORM +ĠV IA +inc r +Ġzen ith +Ġvis cosity +Ġthin ly +.get SharedPreferences +.Error Code +"), " +ĠMillion en +Ġ/> )Ċ +Scroll Indicator +-se eking +ĠPOLIT ICO +as ca +_r l +N avig +(full file +Ġsol itude +Ġju ven +Ġhaul ing +ĠMac ros +ĠG ry +Ġexerc itation +ĠATT ACK +Tick Count +Ġr ites +Ġdo e +Particle System +Ġsl u +Window Text +ĠClass Name +Ġsl ander +ĉ Port +j ong +? a +.D ial +âĢĶ at +$obj PHPExcel +Ġso ar +EN N +appe ared +Ġquot id +em achine +Ġn ip +Ġmicro time +ĠAl ma +; ! +---------------------------------------------------------------- -------------------------------- +ĠPass age +Ġdump sters +ĠEx clude +Ġsuggest ive +ĠCircularProgress Indicator +_cl r +Array Type +ILL A +Elapsed Time +Dr iven +Ġresource Name +ĠG arrison +ser ir +-a head +Ġp innacle +ĠEs presso +S parse +Ġass ays +ĠGirl friend +im id +]=' \ +ONGL ONG +Ġportray ing +L ane +Ġb úsqueda +Ġrein forcements +ĠSpread sheet +ĠArray Collection +, arr +light box +ic ana +< " +build ers +K id +ĠMat SnackBar +EX PR +od cast +ĠFound ations +Ġind s +=' ${ +F izz +-function al +(work space +Ġstem med +_p atches +ĠJar vis +READ ING +Ġdisrespect ful +ĠQ Dom +Ġ$ {Ċ +est atus +Re ached +! .ĊĊ +IL T +ĠN DEBUG +ĠCour age +birth date +ĠT ing +Ġutil izado +án chez +Out door +Ġhand guns +Ref Count +É Ļ +rom o +Ġt ts +.S he +ĠP ane +ãĢij, ãĢIJ +ĠIO CTL +/ black +ins cription +Ġbi opsy +ĠTime Interval +.Test Check +ĠGUI Style +ĠCap ability +ĠBeit rag +don nees +T reatment +.back up +Ġsign ings +ĠB oca +dr m +.M AIN +Ġgo ede +ĠMark up +G REE +ĠBase Service +.C reator +Ġj ails +ĠK ahn +Ip Address +ACH I +Ġinhib ited +Ġ@ $_ +ĠAss ass +Ġenvi ado +Hero es +ÐŁ еÑĢ +ĠM aven +.l s +Ġ ive +| RF +Ġresize Mode +Ġrum pe +_attach ments +T U +Ġtact ile +Attempt ing +Ġro bin +y aw +Ġmerc enaries +ĠHab itat +end date +Ġo xy +ĉR andom +oh on +Is Null +ĠValidation Result +ãĥ ļ +um bed +pp v +Ġar p +ich ick +_r nn +ĠT FT +Tex Image +" On +ĠSam pler +top l +Ġj ane +y ling +ĠUN ICODE +Tab Index +< {Ċ +s uspend +uv ian +, application +ол иÑĩеÑģÑĤво +y at +ez ier +ĠCH UNK +ĠAd ler +/ Add +ĠKey Value +Ġspos ób +Sam pling +ch ers +_AM D +R u +.Must Compile +N ation +Ass oc +Man aging +ĠEng l +_G B +Ġsucc inct +Ġdis liked +ĠI ke +Bullet in +_ARCH IVE +Prop osal +Ġjog ging +.C REATED +Ġch ol +è£ ħ +Į ¨ +-p ush +Ġreserv a +core v +è tre +TH R +Ġincompet ence +Ġchar isma +æĦ Ł +Ġ" == +BT N +ĠLoc ator +iv et +('. ')Ċ +Ġfor IndexPath +ô me +Ġcapac it +w aters +ĠWR ONG +ho a +ĠM IPS +Ġem iss +ĠJacqu eline +(c mp +Ġe ens +Le o +.tim ing +CLUS ION +Ġ(" - +åĵ Ī +.k ode +ĠUnd ert +Ġbew ild +ĠEss en +.h d +Ġren egot +Ġm ower +Ġl sp +Ġpen chant +Ġman oe +Ġag li +Ġrec al +ĠOPER ATION +(^ )( +ĠÎ ½ +ĠSc oped +Ġ@ "Ċ += label +[ loc +Int l +ĠN z +table t +.Column Name +Ġscreen Size +DB us +co oked +- registration +âĢľ One +-n on +ĠwiÄĻ c +Ġcost a +.add Tab +. conditions +ĠH ess +MEM ORY +ĠAval anche +() }}Ċ +Ġtri plet +Ġl abyrinth +ĠNode List +ĠNY T +Ġy eni +d ff +.Html Controls +AV IS +/ Math +Ġmem cmp +Ø§Ø ¡ +оÑģ ÑĮ +c rap +(p ages +Ġl xml +ĠQ DateTime +_t cb +Ġopen id +Ġsyn aptic +ĠMD MA +(s lug +igm atic +en or +Ġcr amped +G OP +Ń IJ +.is File +ĠD ifferential +Ġ=" ";Ċ +ĉĉĉ ĠĠĠĠĉ +ĠC ooke +ĉU FUNCTION +Ġpersever ance +Relative Layout +IMPORT ANT +Ġex on +Ġо н +ib ase +(C ONT +n ovation +ä½ ķ +[ sub +Admin Controller +HTTP Header +cre ar +ĠN IR +ĠDrop DownList +Ġval ide +Ġde hydration +. '] +(W IN +Ġ... \ +Ġphotos hop +ĉ Init +_c ou +Ġtime Zone +dar win +rom atic +Navigation ItemSelectedListener +br ates +] --;Ċ +Ġtraged ies +ĠPed iatrics +SM ART +-A PI +ĠMessage Lookup +ĉ vo +Ġprejud ices +Ġm A +U ps +ĠMISS ING +ĉ ad +C ream +ĠT b +ĠMon a +_ ghost +ĉt ypes +Em b +ĠDocument ary +');ĊĊ ĊĊ +Ġl up +_ Reference +ĠB ATCH +Ġintertw ined +< Cell +ĠCab r +n ation +Ġis Connected +.remove Listener +Ġcon g +_t i +ĠSil icone +Ġê²° ê³¼ +ĠW AN +ĠG ibraltar +/ response +ĉp erson +ch ants +V IP +em ergency +Pixel Format +- Am +Ġsouth western +_pl l +if ers +_ON CE +ĠF ayette +.nc bi +_P anel +.Q ual +Ġpol ys +Ġcreate StackNavigator +� t +Ġlay offs +ĠBl anco +Fe at +ĠV imeo +_ch i +_l ifetime +POINT S +, private +Ġunb earable +print ing +Ġc gi +.B ACK +Ġintern s +ĠNew ly +inf eld +( IB +ĠK ata +ĠDef endants +Th r +é¢ Ħ +_V F +FFFF FFFF +Ġdavid jl +Ġbitter ly +S uggestions +.set Cancelable +FIN AL +ason s +_rw lock +_WRAP PER +Ġhapp iest +(row Index +ós ito +TOT YPE +Autom ation +Log File +Ġcons olation +ãĥ Ģ +Ġt êm +Ġpr er +rg yz +ĠG eg +ĉd to +.default Value +ĠK ami +ĠA SE +optim ized +Ġíı ¬ +Ġorigin ates +err Msg +Ġespa ço +(S YS +ĠMc B +d ance +_det ected +Ġfr ü +ĉĉ ĠĠĠĠĉĉ +< Date +(com b +ĠDec ide +\ Field +ĠProp osed +R ib +Ġdis likes +ĠW ien +ĉ Document +Ġtr af +Ġst oria +ĠT ells +') == +C ri +( VALUE +ĠBurn ett +, void +Ġdan h +Ġc cp +Block chain +:"- "`Ċ +IC lient +IS ODE +Iss uer +) }čĊ +, but +ĠU ph +( Sub +Ġtélé phone +ĠonData Change +Ġmarsh aller +-an alytics +, content +Ġdeb acle +_Value Changed +Ġfa una +Ġ# => +Ġf oyer +'util isation +ĠMü ller +ĠFet ish +Ġdefault Manager +Ġback track +B ah +Exp licit +_A SCII +Ġm Activity +(M sg +Ġê² Į +ĠTER MS +ĠAng ie +HS V +ĠMos que +.N ames +íĬ ¼ +rest e +_p arms +Ġgap ing +Ġcro pping +Data Frame +Ġrespons iveness +_ undo +_tr an +. terminate +Ġitalian e +Ġwalk through +Ġattract iveness +д е +_ST S +_ learn +Ġchocol ates +ier archical +-th inking +Ġ ))) +ish ments +.Log f +ĠTM Z +ĠCan ary +fo il +ĠVacc ine +.v x +ĠSur round +Inter mediate +Ġi ov +v ais +'; ";Ċ +ï½ŀ ĊĊ +éĢģ æĸĻ +âĢ¦ it +Se ats +Cl ar +W ars +ĠHutch inson +ĠHas an +! ')ĊĊ +ĠRich ie +che iden +($ (' +Y ork +Ġl ids +Ġal phanumeric +ĠG lock +.sh apes +Ġspark ing +_ epsilon +uplic ated +.dir ty +]) == +ĠìľĦ ì¹ĺ +Ġsc n +Ġ/ **************************************************************** +_PRE VIEW +_H C +ield ing +f gets +ĠAdd ison +Ġproduct Service +- figure +(ret val +z ano +Ġaut ob +ĉs d +_n umer +ĠSet LastError +ĠF ior +ific ance +Unt itled +Ġin field +Ġ{} ));Ċ +Ġsp ac +Ġro okies +(des cribing +ng en +ி à® +.r df +.M utex +Ġkne eling +ĠQ E +set Max +Read Stream +Ġvent as +s ut +cm peq +.WriteAll Text +ĠEx perienced +$ __ +Ġka um +ĠL IS +Ġdocument os +_HE ALTH +icont ains +Ġart isans +OWN ER +Ġblink ed +get Display +Ġto en +Ġrow Num +Ġav ril +Ġinv is +ĠK ear +toBe InTheDocument +ap ur +Ġr acked +ĠMc Master +_ATTR IB +H az +Ġfact ura +/ ts +ĠÑĢаз меÑĢ +Ġz f +Ġshort fall +.f asta +ĠCONST ANT +.man aged +g ems +Shared Pointer +Ġblur ry +b rightness +( components +Ġ... "ĊĊ +SE LL +ĠIllustr ator +.get Channel +Ġtrou vé +yst ers +Ġvo is +ĠLind en +Ġem ojis +Ġb rawl +ĠMS R +ĠE lo +ĠCroat ian +Popup Menu +L ewis +.J WT +Ġaston ished +B ush +(item Id +Ġdet achment +ĠEnc ore +å° Ķ +Ġre kl +Ġcr am +)$ / +.get Host +_re commend +- HT +_cal ibration +Auth enticate +.firebase app +UN IX +ĉC amera +ĠHE AP +I deal +. office +Ġgoof y +(S ymbol +Ġjou er +_part itions +Ġrapid ement +ĠGN UNET +id User +Ġsuperv ise +( Contact +AW N +ãģ ĺ +Ġna am +Ġa ust +åľ¨ 线 +_soft max +Allow Anonymous +amm able +RO UTE +* D +Ġad en +ĠCrist ina +ĠCrist iano +Ġblood stream +sub class +_person a +CH ILD +-k now +Ġnavigation Options +ĠZuk unft +ĠPix ar +Ty ler +Ġunder world +Ġsincer ity +Ġdispens er +Ġk ter +idd ers +.add Node +- checked +Ġke yst +ĠW TO +.sign als +Ġadvent urer +ĠP ang +\ R += pos +Ġdispens aries +ĠClo set +("{ \" +ide on +Ġnécess aire +() "Ċ +_RECE IVED +Ġrésult ats +Ġmod en +ĠIceland ic +; d +. allowed +(new User +Ġmerc iless +.Wait For +Ġday care +ĠCon veyor +ç ĸ +ð ¬ +ç ĥ +ç Ĺ +ç ł +è Ħ +é ² +å ¦ +çĿ Ģ +å¾ Ī +é ħ +ç ĭ +é ª +æ Ĥ +é ¥ +è ħ +æĥ ³ +å ¨ +é ¹ +ç Ĥ +å Ĵ +ç Į +è´ ¨ +æ ¢ +æ° Ķ +ð « +æķ Ļ +ç Ł +å Ħ +åıij å±ķ +åĪ Ľ +è ij +æ ħ +å ŀ +åģ ļ +æĪ ĺ +æ IJ +å¼ º +æ· ± +åĩ ł +ç ¿ +å © +è ŀ +å§ Ķ +åIJ Ħ +è İ +é ¸ +é º +åı Ĺ +èģ Į +å ĺ +æ ½ +é£ İ +èIJ ¥ +åħ ļ +è ľ +éĤ £ +é¢ Ĩ +ç ij +é ³ +æľ ¯ +ä» Ģ +æĪ ¿ +ç² ¾ +å ª +é Ĩ +å¤ ª +èĤ ¡ +è Ľ +åħ ī +æŀ ģ +åĬ ŀ +è ĵ +ç ĺ +å ´ +å Ĺ +èĬ ± +çł Ķ +å¿ « +å¸ Ī +è¶ Ĭ +è§ Ĥ +æ ¤ +æ ¦ +ç ŀ +èĤ ² +çĪ ± +çĻ ½ +ä¸ ĸ +ä»Ģ ä¹Ī +çľ ¼ +å ³ +è Ĵ +æ ĵ +è¢ « +å¹ ² +çĹ ħ +å£ « +ç Ĵ +è ¸ +æ ¾ +å·¥ ä½ľ +è® © +çĥ Ń +è¾ ĥ +åĦ ¿ +åĬ © +ç§ ¯ +ç ³ +ç ĵ +ç £ +å Ĥ +è ¹ +è ļ +å· ± +çĻ ¾ +åĬ ¿ +èµ Ľ +æ ¨ +æ ¿ +è ĸ +æĿ ij +å¸ ¦ +å¢ ĥ +æĬ ¤ +é Ń +å « +èĩª å·± +æµ İ +ä½ İ +åĮ » +éĺ ² +åĨ ľ +è Ĩ +ç Ĩ +é « +åĨ Ľ +æĪ ı +åį ĩ +æĸ ¯ +ä½ ı +èIJ ½ +åħ » +èĩ ´ +ç Ĭ +ç ĩ +ç ħ +è Ķ +ä¼ģ ä¸ļ +åĽ ¢ +æī į +æł ¡ +åĩ Ĩ +å¥ ĩ +åī ¯ +é ¼ +æ¼ Ķ +é© ¬ +èµ ° +ç¥ ŀ +åħ ĭ +æľ Ľ +æ² ¹ +è¾ ¹ +åį ĥ +å¾ Ģ +åĪ ĩ +æ © +ç ¶ +å Ļ +éĻ ħ +çī Į +社 ä¼ļ +游 æĪı +æĸ ½ +ç ħ§ +æİ § +æ» ¡ +è¯ Ĩ +éĩį è¦ģ +è¶ ³ +çķ Ļ +ç» Ĩ +åį ı +éĢ Ĥ +æ ĩ +æ § +é Ħ +è Ŀ +å¸Ĥ åľº +ç»ı æµİ +ä¹ ł +æĸĩ åĮĸ +éļ ¾ +ä¹ IJ +åĨ ³ +æ¬ ¢ +è§ ī +åĽ Ń +åħ ´ +åħ ħ +ä¸ ¾ +æī ¹ +è ķ +æĬ Ĭ +æĬĢ æľ¯ +ç© ¶ +第 ä¸Ģ +ä¾ ¿ +åĵ į +çİ © +åĿ ļ +èŀ į +åį Ĭ +åĸ ľ +å± Ĥ +ç¦ » +ä» ħ +é Ł +åij ³ +å¿ µ +åŃ £ +ç´ § +ä¹ ħ +é ¤ +é ŀ +è ¤ +åĢ Ļ +åĨ µ +ç Ł³ +åģ ¥ +æĢ İ +å® Ŀ +è¡ Ģ +åŁ Ł +æĹ © +çŁ¥ éģĵ +è´ Ł +åį ļ +å· ´ +äº ² +å± ŀ +ä¸ ¥ +äº ī +å¯ Ł +è º +ç ° +建 设 +产 ä¸ļ +åIJ ĥ +åŃ © +æĹ ħ +æł ¹ +æĿ IJ +ä¼ Ĺ +éļ ı +å® ĺ +åº ķ +å½ © +å¯ Į +æ¸ © +åį « +åī § +çĽ Ĭ +æĬ Ĺ +è´ ¢ +çº ª +æ Ĩ +çĶŁ æ´» +çº ¢ +çĶŁ 产 +è¿ ľ +éĴ ± +åĶ ® +ç¾ ¤ +çı Ń +æ¥ ¼ +éĩ ĩ +èī º +å± ħ +åģ ĩ +è° Ī +æĻ ļ +é ¬ +èĪ ª +å® ³ +è Ĺ +ç į +å µ +çİ ĭ +åº · +è İ· +ç» Ń +äº ļ +é£ Ł +åİ ĭ +æĭ Ľ +èĮ ĥ +è® ¸ +åĽ ´ +é ½ +éĻ į +çº ³ +åĵ ª +æķĻ èĤ² +å·² ç»ı +å¾ · +æŀ Ĺ +å®ī åħ¨ +é¾ Ļ +大 家 +éĿ Ĵ +åº ľ +æ² ³ +åı ¤ +èį ¯ +åĿ ĩ +æĻ º +ä¹ ¡ +çķ ¥ +åĨ · +ç¦ ı +å® ¤ +ç» ´ +æī ¿ +å± Ĭ +è¯ ī +åĪ » +è Ł +æ ª +å°± æĺ¯ +è¿Ļ 个 +ä¸Ń å¿ĥ +ä¸ĸ çķĮ +åŁİ å¸Ĥ +éĿŀ 常 +åĪ Ĵ +åı Į +æĢİ ä¹Ī +åĪ° äºĨ +æľ ĥ +åı ² +ä¾ Ĩ +å¾ ĭ +å¥ ĸ +ç» Ī +åª Ĵ +å® ģ +è¯ ¾ +èģĮ ä¸ļ +åħ į +æµ ĭ +æĢ ¥ +æķ ij +çĭ ¬ +èŃ ¦ +é¤ IJ +æĦ ¿ +è´ « +çĸ ij +å ļ +å¥ ¹ +åı Ī +åĽł 为 +ä¸į æĺ¯ +å¤ Ł +æĸ¹ éĿ¢ +éķ ĩ +äº Ĵ +éħ Ĵ +è® ² +çĸ Ĺ +æĺ ¥ +æ¹ ĸ +å¤ ľ +è´£ ä»» +人 æ°ij +åħ ° +çŁ Ń +æķ ħ +åĩ ı +æĻ ® +äº ® +ä¾ Ŀ +åį ° +éĿ Ļ +åĢ ĭ +å¾ ģ +åIJ ¸ +ç¼ º +æĶ » +åĩ Ģ +åħ ¸ +åĽ º +è® ¿ +ç ¹ +ç Ģ +æıIJ ä¾Ľ +ç» ĩ +å¾Ī å¤ļ +çłĶ 究 +è· Ł +主 è¦ģ +æĥħ åĨµ +çŃ ĸ +æŃ » +大 åѦ +æĶ¿ åºľ +å½± åĵį +ä¹ ° +åħ Ń +éĻ © +åħ « +æŁ IJ +è´¨ éĩı +åį ł +å· ® +æĽ´ å¤ļ +æľ ĭ +éĿ © +å® £ +çł ´ +è½ » +åº § +æĺ ¾ +ç¨ ³ +è´ µ +èĥ Į +èī ¯ +çĸ « +æ¯ Ĵ +ä¹ İ +åĢ Ł +è¿ · +çŃ Ķ +æ¿ Ģ +åij ¼ +äºĨ ä¸Ģ +è¶ £ +ä¼ ´ +ä¼ Ļ +è ¼ +ð¬ Ń +åĽ½ 家 +æ´» åĬ¨ +çİ° åľ¨ +ç§ij æĬĢ +åį ¡ +ä¸į åIJĮ +个 人 +è®° èĢħ +ä¸į æĸŃ +éĹ » +ä¹ Ŀ +èij Ĺ +ç» ¼ +ä¸ ĥ +æł ij +æľĭ åıĭ +åį ĸ +ä¼ ¤ +æ² Ļ +åĸ Ħ +å¥ Ĺ +è½ ® +ç© ¿ +è¡ ¥ +ä¸Ģ å®ļ +çª ģ +çĿ £ +è¿ ½ +å¨ ģ +åı ¦ +åĽ ° +æŀ ¶ +ç» Ŀ +æķ £ +æİ ¢ +æ´ Ĺ +ä¸ ´ +ä¼ ¼ +è´ ¸ +ä¸ ° +æĺ¯ ä¸Ģ +ç« ŀ +è¿ İ +èģ ļ +è « +æį Ł +æī § +é© ¾ +è¿ Ŀ +è ¥ +è ł +ä»ĸ 们 +æŶ åĢĻ +å® ĥ +人 åijĺ +è¿Ļ æł· +å·¥ ç¨ĭ +åĪĽ æĸ° +åŃ© åŃIJ +å¸ Į +éĥ¨ åĪĨ +éĵ ¶ +代 表 +é¦ Ļ +å¸ ® +æݨ è¿Ľ +çĽ ĺ +积 æŀģ +éĥ¨ éŨ +åŁ ¹ +æŃ ¦ +ä¸į ä¼ļ +çŃ ij +éĢ Ļ +çİ© 家 +æĭ ¿ +åİ Ĥ +æ¯ Ľ +çģ µ +æŃ Į +ç »¿ +å¦ Ī +çĽ Ľ +é¦ Ĩ +é¡ º +èĦ ¸ +å° ¼ +ä¸ ½ +å¥ ¥ +éģ ĩ +è¯ į +å° ģ +ä¸ Ŀ +好 çļĦ +æĭ ħ +èĦ ± +æģ ¶ +åİ ļ +åĬ ³ +çĽ Ł +æĬ ĺ +åı ¥ +æĢ Ģ +æŁ ĵ +书 è®° +åĨ ł +é² ľ +æ ¦Ĥ +éļ IJ +å¹ ħ +èµ ŀ +å¹ ķ +æ¥ Ń +éģ Ĺ +åĪ ¤ +è ĺ +å ¶ +æĬķ èµĦ +è¡Į ä¸ļ +äº ij +çݯ å¢ĥ +åѦ çĶŁ +åIJĪ ä½ľ +åģ¥ 康 +é£ ŀ +ä¸Ģ æŃ¥ +ä¸Ģ 缴 +åıij çĶŁ +éĺ ¿ +é¢Ĩ 导 +åĸľ 欢 +åºĶ 该 +çĤ º +è® Ń +æĿ Ģ +æ¸ ¯ +交 éĢļ +éĺ ¶ +éĴ ¢ +ä» ¤ +å° ½ +æ¯ į +è¡ £ +ç² ī +é¡ ¶ +ä¹Ł ä¸į +æĬ ĵ +èĭ ¦ +å¹ ¸ +ç¤ ¼ +第 ä¸ī +大 çļĦ +éģ İ +çĥ Ł +éģ ¿ +ä» į +åº Ĩ +æĢ ķ +è° ¢ +çĽ ĸ +å° Ħ +éľ ² +æĸ Ĺ +ç Ĭ¶ +åŃ ¸ +æ¯ ķ +å· ¨ +çŁ ¿ +çļ ĩ +å¸ Ń +çĹ ĩ +æī ¬ +å» ¶ +ä¾ § +æ· ¡ +çļĦ ä¸Ģ +ç¶ ² +æ´ ģ +ç ¸ +è§ Ī +çŃ ¹ +ç§ ĺ +è¯ Ĭ +çı ¾ +èª ī +æ¯ « +ð ¨ +åį ´ +æĪIJ 为 +èĥ½ åĬĽ +é» Ħ +æĹħ 游 +èĪ ¬ +æ¯Ķ è¾ĥ +èµ· æĿ¥ +äºĨ 解 +èĩª çĦ¶ +ä¸Ģ 次 +åŁº æľ¬ +æĽ ¾ +综 åIJĪ +èı ľ +è§ī å¾Ĺ +第 äºĮ +è· ij +æ³ ¢ +åĢ Ĵ +ç¡ Ģ +åħ µ +èį ī +çĶ ³ +çĶ ° +æĤ £ +è§Ħ å®ļ +èĥ ľ +èµĦ 产 +æ¢ ¦ +æľ Ŀ +è¿Ļ éĩĮ +å¤ « +æĮ ¥ +ä½ Ľ +å® Ī +éĽ ¶ +æĸ ¼ +ç¯ ĩ +å² Ľ +åĵ ¥ +éŃ Ķ +ä¸į åĪ° +æī ĺ +åº Ĭ +æ¬ § +èį £ +æ± ĩ +æī © +åģ ı +å¢ Ļ +è® ¯ +å© ļ +æĥ ł +æ´ ĭ +å® ľ +æ¶ ¦ +æħ ¢ +éĢ ı +å® ½ +é¡ ¾ +ç´ ¯ +æ± ¡ +çĪ Ĩ +ç§ Ł +æĥ Ĭ +æ¶ ¨ +é¥ ° +éĺ µ +é¥ ® +æļ ĸ +åº Ł +æĹ Ĺ +éļ Ķ +ç¶ ĵ +åĭ Ļ +å¯ ¦ +éĢ Ķ +æī « +çĥ Ī +éĽ » +åĪ ij +éĹ ľ +éĹ ª +å¥ ĭ +å Ĥ¨ +ç¼ © +ä¾ µ +å ¬ +𬠶 +åĽ½ éĻħ +ç»Ħ ç»ĩ +ä¸ĵ ä¸ļ +åıij çİ° +å¸Į æľĽ +ç»ı èIJ¥ +åı « +æĿ¥ 说 +éļ ľ +ä»» ä½ķ +交 æĺĵ +éĩį çĤ¹ +çļ ® +ç» į +æ´ ¾ +ç§ij åѦ +åºĶ çĶ¨ +建 çŃij +èĤ ī +æĶ¹ éĿ© +åŁº ç¡Ģ +æ± ī +åĩº æĿ¥ +è¿Ļ ä¹Ī +åĪ ļ +åĿ IJ +ä¸į ä»ħ +ä¼ļ è®® +éĿ ł +åªĴ ä½ĵ +æ° ¸ +åĨ ² +èĭ ı +å¤ ® +çĪ ¶ +åł Ĥ +å®ŀ éĻħ +è¡ Ĺ +ç« ¥ +éĺ ħ +äºĭ æĥħ +åİŁ åĽł +éħ ¸ +以 æĿ¥ +å¨ ± +å® « +åĿ Ĺ +ç» © +éĩ İ +ä¸į å¾Ĺ +ä¼ł å¥ĩ +ç¡ ¬ +åİ ħ +æĹ ¢ +ç» ĥ +èĦ ij +å¼ ± +æİ Į +è´ ´ +æĮ Ĥ +åħ³ éĶ® +å° ļ +é¥ Ń +åº Ħ +çĻ ¼ +åľ ĭ +æİ Ī +个 æľĪ +äº Ī +å¸ ģ +è· Ŀ +æ² ī +ç« Ł +åĨ ¬ +æĬ ½ +éĨ Ĵ +å¼ Ł +è§ ¦ +èģ ĺ +è± Ĩ +æļ ´ +åijĬ è¯ī +è± ª +èµ ¢ +è· ¨ +è³ ĩ +çĪ ¸ +æĬ ± +æµ ª +éº » +ä» ª +è¡ ¡ +å¥ ¶ +çģ ¾ +èµ ¶ +èĤ ¥ +å§ IJ +åĢ º +éľ ĩ +è® ¢ +æ¬ Ĭ +ç · +å» ī +ä¿ Ĺ +å¿ ĺ +å¦ ĩ +ç¼ ĵ +åŃ ķ +æ¼ « +è£ ģ +çĩ ĥ +é» ĺ +çī ¢ +çĪ · +æĬ µ +å® ¾ +æľī ä¸Ģ +è¿ ¹ +è¿ « +è² Į +æľī çļĦ +ð¬ ĺ +è¿ĺ æĺ¯ +æīĢ 以 +ä¹Ł æĺ¯ +è¿Ļ äºĽ +对 äºİ +åIJ § +缮 åīį +èĩªå·± çļĦ +èĥ½ å¤Ł +å¦Ĥ ä½ķ +æľº æŀĦ +åıª æĺ¯ +ç½ij ç«Ļ +åħ¨ éĿ¢ +为 äºĨ +å¼Ģ åıij +æĸ° éĹ» +éĩij èŀį +ç» § +客 æĪ· +ä¸Ģ èµ· +èĮ ¶ +åħ³ 注 +æ°´ å¹³ +åİĨ åı² +å¢ŀ éķ¿ +é ± +åŁº éĩij +åº Ń +åı ¶ +ä¿ ĥ +éĽ ¨ +æ¶Ī è´¹ +èĪ ¹ +çŁ¥ è¯Ĩ +æĪĺ çķ¥ +ç»ı éªĮ +å³ ° +æĽ ² +èĦ ļ +åĨ ° +å¤ ı +å½ Ĵ +ç¬ Ķ +èĻ ij +çĶ ² +åľ Ī +è¯ Ĺ +é½ IJ +容 æĺĵ +çłĶ åıij +éª ¨ +çº ¸ +è· µ +æĹ § +çķ ¶ +åĪ ¸ +è´ · +åı ¬ +ç§ ĭ +æ¶ ² +è¡Į æĶ¿ +çĮ ® +èĤ ¤ +éĢ IJ +è¶Ĭ æĿ¥ +è¶ĬæĿ¥ è¶Ĭ +æĦı è§ģ +èĪ ŀ +åī Ĥ +æ¶ ī +ç¨ĭ 度 +åħ¬ åħ± +æ¢ ° +æľ « +çº ¯ +åĶ ± +æ´ ² +æĬ ¢ +æ¤ į +å¿ Ļ +ä¼ ° +å¼ ¹ +æ³ ī +æľĢ 大 +è¶ ĭ +å· § +ç¦ ģ +æī ¶ +åį ± +çı ł +çĨ Ł +æĭ ľ +主 ä¹ī +æĿ Ĥ +éĻ Ħ +éģ į +æIJ Ń +æĮ ¯ +å¤ļ å¹´ +æķ ¬ +æij Ħ +çº · +å¼ ĥ +æ¹ ¿ +å¨ ĺ +æ¡ £ +é© ¶ +æľ Ĺ +æ® ĸ +æ¦ ľ +åĵ ¡ +ä¸Ģ ä½ĵ +æŁ¥ çľĭ +ç¹ ģ +æµ ĵ +åħ¬ å®ī +æ½ ľ +è´ ¯ +éª Ĺ +æ IJľ +å· ¡ +è ¬ +é Ĭ +å§Ķ ä¼ļ +æĤ ł +åī © +æı Ń +åŃ£ 度 +ð «ĺ +𬠬 +ä ´ +ð ª +ä½Ĩ æĺ¯ +éĥ½ æĺ¯ +å¹³ åı° +åѦ ä¹ł +åĵģ çīĮ +ä¸ Ķ +è¿Ļ ç§į +æĶ¿ çŃĸ +æĭ ¬ +认 为 +ä¸Ģ èĪ¬ +æłĩ åĩĨ +æĶ¯ æĮģ +模 å¼ı +åħ³ ç³» +çļĦ æĺ¯ +è¿Ļ ä¸Ģ +ä¸į è¦ģ +çĶ ļ +ç²¾ ç¥ŀ +æĭ ¥ +åĪ© çĶ¨ +ä¿Ŀ æĬ¤ +ä½ľ çĶ¨ +èĭ ¥ +åĽ½ åĨħ +ä»ĭ ç»į +ä¸Ģ ä¸ĭ +å·¥ ä¸ļ +缮 æłĩ +æľĢ åIJİ +ä»· åĢ¼ +å° į +éĵ ģ +è° ģ +ç»ĵ æŀĦ +éĽ ª +æĻº èĥ½ +ä¼ł 绣 +ä½ĵ èĤ² +çĶŁ æĢģ +æĭ į +æİ ª +åĨľ ä¸ļ +çī¹ èī² +è§Ħ 模 +æŶ 代 +è¿ĩ ç¨ĭ +éĴ Ī +æĿ ¾ +åĶ IJ +åĮ» çĸĹ +çģ ¯ +åĪ¶ éĢł +æł¸ å¿ĥ +ä¸į åı¯ +ç³» åĪĹ +åIJ ī +åľ £ +åĢ ij +ä½ ³ +æĿ¥ çľĭ +æ¯Ķ èµĽ +ä¸ĭ æĿ¥ +åĩº äºĨ +å¹² éĥ¨ +å¾® ä¿¡ +å½ĵ åľ° +åį · +åį« çĶŁ +ä¼ Ł +çĸ« æĥħ +è° · +åĩł 个 +éĺ ´ +çĶŁ çī© +å° ¤ +ä¼ Ĭ +èĤ ¯ +éĿ¢ 积 +åĪĽ éĢł +æı ¡ +åľ Ĩ +æĻ ĵ +æĪIJ äºĨ +åĩ ¡ +çĸ ¾ +ç«ŀ äºī +è® ¨ +主 é¢ĺ +é² ģ +è¿ ª +ä¿ Ħ +æĢ ª +ä¸ ¦ +èĻ ļ +æ½ ® +çĥ § +èĢ ³ +æ± ł +éĢĤ åIJĪ +æł¹ æľ¬ +åĬł 缣 +çĶµ è§Ĩ +æ· · +ç¼ ĺ +çª Ĺ +çĬ ¯ +æĥ ¯ +æĦı ä¹ī +åĬŀ æ³ķ +ä¼ ij +æ» ij +åĭ ĩ +æķ ¢ +å¯ » +è¦ Ĩ +éĢ ĥ +ç»ı çIJĨ +åĿ ı +æ³ ½ +ä¹ ĺ +åĪ º +å± ı +é¡ ¿ +äº ¡ +éĤ Ģ +åħ ¼ +åĭ ¤ +æ® ĭ +æĺ ł +æ¯ķ ä¸ļ +æĪ ª +è· Į +å£ ģ +åı¦ ä¸Ģ +羣 å®ŀ +ç£ ¨ +è¯ ļ +å¿ħ è¦ģ +æģ ĭ +æĩ Ĥ +å¾ Ĵ +è° ĵ +æķ ı +æ Ļ¨ +èĥ ¸ +æĭ ¼ +å¦ Ļ +è¯ ¸ +èģ Ĭ +æĤ ī +éº ¼ +åĩ Ń +èĪ Ĵ +æ¶ Ĥ +è¿ ģ +æ² ¿ +å¡ ij +æĽ ¿ +æ¾ ³ +å¿ į +èĢ Ĺ +éľ ¸ +åĩł å¹´ +åĪ Ĭ +èĦ ī +èħ IJ +æ¡ Į +çº ł +æ» ļ +æĤ ² +åĨ Ĵ +å¦ ¹ +çķ ħ +çº µ +æij ĩ +å¤ º +è·¯ ä¸Ĭ +å¿ ½ +èĸ ª +æģ IJ +æĦı æĢĿ +å« Į +æı ´ +æ° § +èĢ Ģ +éĺ » +è½ ¨ +å¹ » +æį ķ +åĿ ¦ +åĵĪ åĵĪ +çĭ IJ +æ» ¨ +è² » +è¿ Ł +人 éĥ½ +ç» ĺ +åı ¹ +çµ IJ +æī ° +æ» ĭ +å¥ ij +åĭ Ł +ç¢ º +ð ¦ +éĽĨ åĽ¢ +æĿ İ +å¼Ģ å±ķ +æıIJ åįĩ +åħ¨ åĽ½ +æ±½ 车 +åѦ æł¡ +æł¹ æį® +è¿Ļ æĺ¯ +åĩº çİ° +éĻ Ī +ç½ Ĺ +èİ· å¾Ĺ +åĪ ĺ +éĶĢ åĶ® +æľª æĿ¥ +éľĢ æ±Ĥ +å®ŀ æĸ½ +åĿļ æĮģ +åħ¨ çIJĥ +éĵ¶ è¡Į +æݧ åĪ¶ +é¡ » +åľ° åĮº +æīĵ éĢł +çļĦ è¯Ŀ +帮 åĬ© +ä½ĵ ç³» +è¾¾ åĪ° +è§Ħ åĪĴ +åŁ¹ è®Ń +两 个 +æĬ¥ åijĬ +åľ° æĸ¹ +å®Į åħ¨ +æİ ī +ç»ĵ åIJĪ +宣 ä¼ł +æ³ķ å¾ĭ +èīº æľ¯ +çĶµ å½± +èª ª +ä¸Ģ çĤ¹ +è¶ħ è¿ĩ +çĶµ åŃIJ +æĢĿ æĥ³ +æķĻ åѦ +éĺ¶ æ®µ +åķĨ ä¸ļ +çī© æµģ +åĪĽ ä¸ļ +æĸ¹ æ¡Ī +çİ° 代 +æ¡ ¥ +èIJ½ å®ŀ +带 æĿ¥ +产 çĶŁ +ç§ Ģ +æ³ ° +ä¹ ± +åħ· ä½ĵ +åĸ Ŀ +èĵ Ŀ +å® Ĺ +åįĩ 级 +æ·± åħ¥ +ä¿Ŀ éĻ© +ç®Ģ åįķ +çĹ Ľ +稳 å®ļ +è¾ Ĩ +å±ŀ äºİ +å· Ŀ +ä¸į å°ij +åĴ ¨ +举 西 +å½¢ å¼ı +娱 ä¹IJ +æŃ£ 常 +é¸ ¡ +åħħ åĪĨ +å®ŀ è·µ +éĩĮ éĿ¢ +è· ³ +èĻ İ +æĪIJ éķ¿ +æļ Ĺ +çĿ ¡ +ç½ ª +çIJĨ 念 +æĮ ij +èµĦ æľ¬ +å¤ļ å°ij +ä¸ĭ éĿ¢ +å¸ Ŀ +åħ¬ å¼Ģ +æ¸ IJ +éķ · +å± ĭ +欢 è¿İ +å¿ĥ çIJĨ +çĤ İ +æ¹ ¾ +è® ĵ +éĤ Ħ +ç³ ĸ +ä¹ Į +åĬ ± +çī Ļ +èħ ¿ +å² Ĺ +ä¼ į +æĪIJ åijĺ +åŃ Ķ +å°ı ç¼ĸ +èij £ +æ³ ¡ +åħĪ è¿Ľ +åħ § +åĺ ´ +è´ Ŀ +è » +æIJ ŀ +æ³ Ľ +é¸ Ł +ç½ ² +èĽ ĭ +主 ä»» +缮 çļĦ +ä¹ ı +æ´ ¥ +æĪ ´ +严 æł¼ +çħ ¤ +çĮ « +åĶ ¯ +å° Ĭ +çĶ ľ +åŀ ĥ +åľ ¾ +æĭ Ł +çĦ ¦ +é« Ķ +å® ı +æ© Ł +é© » +æĹ ģ +å½ » +éĥ½ ä¸į +æij © +ä» ĵ +ä¹ ³ +å² ¸ +è° ĭ +大 å¤ļ +çģ Ń +èħ ¾ +æŁ ľ +èĪ į +åħļ çļĦ +å° ĺ +åįģ å¹´ +æĭ Ĵ +è£ ¡ +æŁ Ķ +å¹ ¼ +éĶ ģ +ä¸ĵ 项 +æī İ +驾 驶 +ç¢ İ +è¢ ĭ +éĶ ĭ +å£ ® +å° ĸ +çĶµ æ±ł +è¿ Ķ +æ¼ ı +å¾ ª +èı Į +èĥ ĥ +è¾ ħ +éĢ Ĵ +èĥ İ +éĻ ª +å¯ ¿ +å¥ Ķ +çĮ Ľ +çº ¹ +çŁ¥ åIJį +å¿ Ĩ +æ¡ ĥ +æ£ ĭ +éĢ Ĩ +çĤ ¼ +ç± į +çī § +æł· çļĦ +è¾ Ľ +åł Ĩ +å®ŀ åľ¨ +ä¼ ı +å® ¿ +èµ ı +è£ Ĥ +åįĬ å¹´ +åĢ ¾ +满 æĦı +æ¢ ¯ +æĦı åij³ +åŃ ¤ +ç¥ Ŀ +æĻ ¶ +èµ Ķ +åģ ¿ +èĦ Ĥ +ç½ ļ +ç¢ į +æ² ĥ +æ ĵį +å´ ĩ +æļ Ĥ +è· ĥ +æIJ ¬ +å© Ĩ +é ī +éī ´ +åħ´ 趣 +èIJ¥ ä¸ļ +è® Ĭ +èĦ ı +è¾ Ī +å·ŀ å¸Ĥ +è´« åĽ° +ç© · +ä¸Ń å°ı +æ¼ Ĥ +çĻ Į +èľ ľ +ä¼Ļ ä¼´ +çī µ +æĤ Ł +éĻ · +èµĽ åŃ£ +æ¨ £ +åģ ¶ +æĺ Ĩ +è¢ Ń +æį IJ +èī ° +æ Ĥ¬ +çĶ ¢ +èij ¡ +çĽ Ĺ +å© ´ +å° İ +çº ½ +åĢ ¡ +æī ® +è¨ Ń +æĬ ij +ç¡ ķ +è¾ ĸ +éĥ ģ +è¾ © +éĤ » +çİ° åĩº +è¦ ı +å½ ¹ +éĺ Ķ +åī µ +è¯ ± +æĥ ij +æ· Ģ +é¢ Ī +ä¾ ¦ +æģ ° +æ£Ģ å¯Ł +éĨ « +çĦ¶ æĺ¯ +åĭ ĥ +èĮ « +ä ĵ +𠬸 +ä½ľ 为 +çļĦ 人 +éĤ£ ä¹Ī +ç¾İ åĽ½ +è¿ĺ æľī +æıIJ é«ĺ +èĻ ½ +åħ· æľī +åĮħ æĭ¬ +æĪĸ èĢħ +ä¸į è¿ĩ +ä¸Ĭ æµ· +åĮ» éĻ¢ +èµĦ éĩij +çĶļ èĩ³ +åĪ¶ 度 +解 åĨ³ +èģĶ ç½ij +继 ç»Ń +建 ç«ĭ +è¿Ľ ä¸ĢæŃ¥ +æĿIJ æĸĻ +ä»Ĭ 天 +å¿ħ é¡» +åIJĦ ç§į +çİ° åľº +ä»ĸ çļĦ +å¢ŀ åĬł +é¢Ĩ åŁŁ +åıĤ ä¸İ +æĮģ ç»Ń +ä¹ĭ ä¸Ģ +çī¹ åĪ« +é± ¼ +åħ± åIJĮ +åĬ ª +çİ ī +人 们 +åħĪ çĶŁ +ä¼ĺ åĬ¿ +ä¿Ŀ æĮģ +ä½ľ åĵģ +çī Ľ +æĪIJ æľ¬ +æĶ¶ åħ¥ +åıĬ æŶ +è´Ł è´£ +æİ¥ åıĹ +èį IJ +åıª è¦ģ +羣 çļĦ +导 èĩ´ +æľº åĪ¶ +è¡Į åĬ¨ +æĸ° çļĦ +å®Į åĸĦ +为 ä»Ģä¹Ī +ä¸Ń 央 +æĪIJ ç«ĭ +æĦŁ è§ī +åıĺ åĮĸ +åıĹ åĪ° +并 ä¸į +åŃ Ļ +æĸ½ å·¥ +æĺİ æĺ¾ +è¿ĩ åİ» +åıij æĮ¥ +羣 æŃ£ +åŁº åľ° +æĺİ ç¡® +èĥ ¡ +许 å¤ļ +ä¸Ģ å¹´ +æĸ¹ åIJij +æģ © +缸 ä¿¡ +åľ ³ +详 ç»Ĩ +äºĭ ä¸ļ +çĶŁ åij½ +åĴ¨ 询 +æĸĩ æĺİ +çij ŀ +绿 èī² +èİ « +æĦı è¯Ĩ +æĬķ åħ¥ +åĬł å¿« +æ¢ ħ +ç¿ » +å¼Ģ æĶ¾ +æĻ® éĢļ +åįı ä¼ļ +æĪIJ 绩 +ä» Ļ +å¯ Ĵ +è¯ģ åĪ¸ +认 è¯Ĩ +ä¸ ¹ +大 éĩı +è¿ ħ +åģļ åĪ° +设 æĸ½ +è´¸ æĺĵ +èĥ½ æºIJ +æŶ æľŁ +ä¸Ģ 天 +æ²» çIJĨ +åĺ ī +å® ĩ +丰 å¯Į +举 è¡Į +æĪIJ æŀľ +èĤ¯ å®ļ +çĭ Ĺ +åĬ¨ åĬĽ +æ£ ® +åĩł ä¹İ +åĽł ç´ł +æ°ij æĹı +æ´ ŀ +ç½ij åıĭ +åIJĪ çIJĨ +广 大 +æ® Ĭ +æ´ Ľ +æĿ ¯ +èĴ Ļ +çĶ¨ äºİ +èŀį èµĦ +ç¥ ĸ +æľº 械 +举 åĬŀ +èĩª åĬ¨ +åĬŀ åħ¬ +é» ŀ +éĽ Ħ +åĢ¼ å¾Ĺ +çĮ ª +以 为 +æĺ Į +è·Ŀ 离 +åIJ¸ å¼ķ +ç» ķ +éļ Ĩ +计 ç®Ĺ +éĺŁ ä¼į +大 ä¼ļ +å¼ķ èµ· +çī¹ çĤ¹ +èĥ ¶ +å¹´ è½» +æľ¬ 身 +æľº åħ³ +å®ĺ æĸ¹ +éĥ ij +æµ Ļ +è§Ĵ èī² +èij£ äºĭ +为 主 +æĹł 论 +ä¹ł æĥ¯ +æ¥ ļ +æĭ ĵ +绣 计 +åħ Ħ +广 æ³Ľ +åį Ģ +污 æŁĵ +è« ĭ +èĬĤ 缮 +ä¼ ¦ +è¦Ĩ çĽĸ +èĢ IJ +æī¶ è´« +ç»ı åİĨ +éĩįè¦ģ çļĦ +èĤ¡ 举 +æĭĽ èģĺ +åĽĽ 个 +æĩ ī +èĥ ŀ +æij Ĩ +é«ĺ éĢŁ +éº ¦ +åİŁ åĪĻ +èİ ± +æĽ´ 好 +éķ ľ +åĩ Į +åŀĥ åľ¾ +éĢ ² +çģ ° +éĵ º +äºĭ æķħ +çĶ ĺ +空 æ°Ķ +é¾ Ħ +èı ² +çĵ ¶ +æĺ ¨ +æĹ¥ æĬ¥ +æµ ® +åľ° åĽ¾ +åij Ī +大 åĬĽ +ç» ª +å¸ ħ +æľį åĭĻ +ä¸į éĶĻ +乡 æĿij +å± ¥ +å¹³ æĸ¹ +éĹ ² +æī £ +ç´ł è´¨ +èµ ´ +éģ Ń +èIJ ¨ +èĩª 主 +éĩij å±ŀ +èī¯ 好 +两 å¹´ +æ³ ¥ +é¢ ľ +ç²¾ 彩 +ä¸Ń åįİ +æĻ ĭ +ä¹ł è¿ij +ä¹łè¿ij å¹³ +æĪĺ 士 +åģļ çļĦ +éª ij +æ» ´ +çĵ ľ +çīĪ æĿĥ +èĤ ł +æľĥ åĵ¡ +çı į +ç¨ ® +ä »¿ +çī© ä¸ļ +åĢĭ 人 +å¦ » +ä¼ ¸ +æ± Ĺ +æĹ º +çIJĨ æĥ³ +æij ¸ +è¿Ŀ æ³ķ +å®Į æķ´ +åİ ¦ +è¸ ı +æĸ ij +æ¡ Ĥ +ä½ĵ åĪ¶ +å¸ « +æĿ Ĩ +æ® ¿ +æ¯ ģ +é¦ Ī +è§Ĵ 度 +æ¬ £ +çĥ ¦ +èĤ º +éĩĩ 访 +æij ĺ +æĮ ¡ +æ· ĺ +åħ» èĢģ +çĤ ¸ +è¿ Ī +åİ ī +åĿ Ĭ +è¾ £ +åĩ Ŀ +æ³ ª +çĸ ı +æİ ĺ +åĥı æĺ¯ +éĽ ķ +ç¼ Ŀ +èį · +æį · +åł ¡ +åı¥ è¯Ŀ +çĸ ¼ +æł ı +éģ µ +ç¢ ³ +å·¥ åķĨ +æIJ º +åĪ ¥ +ä¹ Ļ +æĹ ĭ +æĥ ľ +ä¸Ģ 大 +å±Ĥ 次 +èµ ĸ +æĬ ¬ +æ¨ Ĥ +è¯ ŀ +åħ Ĵ +ç¯ ® +èĤ ĥ +å§ ¿ +æĬ ļ +çĵ · +çĶµ åĬ¨ +æĸ° åĨł +æ¶ µ +ç¢ ij +æ· ® +æĹ ¨ +è¸ ª +æ¸ Ķ +æĦ Ī +åı Ķ +åįĹ çľģ +ç¾ © +å§Ķ 书记 +è² ¸ +æ¶ Į +è« ĸ +èIJ Ħ +æı ı +å¿ § +è¾ ¦ +å¦ Ĩ +æī Ń +åij µ +éģ ¥ +è¨ ± +ä» ĩ +åįģ ä¸ī +åī ² +èª į +èĪ ° +é¢ ĩ +é¥ ± +çĭ ł +é«ĺ çļĦ +çµ ± +æħ İ +é¢ ģ +åIJĪ éĢĤ +æµ ´ +èµ ĭ +æĬ ¼ +å¦ ¥ +éĻ¢ éķ¿ +èĢ ķ +è¾ ¨ +æħ ° +åįģ åĽĽ +æľ µ +èĵ Ħ +æŀ ¢ +å» · +æĤ Ħ +æ¶ ¯ +çŁ © +åŃIJ éĩĮ +çĬ ¹ +å±Ģ éķ¿ +é IJ +å¥ ł +ä¼ļ éķ¿ +æĵ ļ +ä¸į åıĬ +åįģ ä¹Ŀ +æ¬ º +èº º +éĺ IJ +çº Į +è¨ » +åĨ Ĭ +èŃ ĺ +é«ĺ çŃī +èħ º +å¤ ķ +ç» ij +åĶ ¤ +èķ ´ +çķ ľ +æħ ĭ +åı Ļ +åı ĥ +å³ ¡ +人 大 +éħ ¿ +éģ © +å¥ ¢ +åı£ æ°Ķ +éĮ Ħ +é ı +åĭ ĺ +è´ ¿ +éļ ª +é ĭ +éļ ¶ +ð ¥ +𬠣 +ð £ +ð« į +𬠳 +ð« ĵ +ð« Ħ +ð« Ł +𨠱 +ä Ĺ +以 åıĬ +æľī éĻIJ +åij ¢ +åIJ Ĺ +çľĭ åĪ° +计 åĪĴ +è¿Ľ åħ¥ +缴 æİ¥ +åĪĨ æŀIJ +åıª æľī +设 å¤ĩ +åħ¶ å®ŀ +åĬł 强 +ä¸Ń çļĦ +ä¿Ŀ éļľ +èĢģ å¸Ī +人 æīį +å¾Ĺ åĪ° +é£İ éĻ© +ä¸Ģ ç§į +空 éĹ´ +æĪij åĽ½ +ä¹ĭ åīį +ä¸ĵ 家 +æĿ ¨ +æĹ¥ æľ¬ +群 ä¼Ĺ +åıĤ åĬł +æķĪ æŀľ +æľī åħ³ +家 åºŃ +åĮº åŁŁ +åĬª åĬĽ +éļı çĿĢ +æĹł æ³ķ +交 æµģ +è¡Į 为 +æ£Ģ æŁ¥ +æľŁ éĹ´ +å¦Ĥ æѤ +èĤ¡ 份 +å½ĵ æŶ +è£ħ å¤ĩ +åĩĨ å¤ĩ +éħĴ åºĹ +è¿IJ åĬ¨ +æıIJ åĩº +å·¦ åı³ +æݪ æĸ½ +é£Ł åĵģ +æ¶Īè´¹ èĢħ +åѦ éĻ¢ +æĮĩ 导 +è¿IJ èIJ¥ +éĩį 大 +åĨľ æĿij +éĢł æĪIJ +æĶ¿ æ²» +éĴĪ 对 +æŃ£ å¼ı +åıĸ å¾Ĺ +éĤ£ 个 +éĽĨ ä¸Ń +åıª èĥ½ +å¿« éĢŁ +身 ä½ĵ +åħļ åijĺ +èģĶ åIJĪ +åĬĽ éĩı +éĥ½ æľī +æ ħ§ +å¡ Ķ +åĪ« 人 +表 çİ° +æķħ äºĭ +ä¸Ģ åĪĩ +å° ĩ +èµĦ æĸĻ +åŁ¹ åħ» +éĺħ 读 +æľī 人 +èIJ¥ éĶĢ +çĽij çĿ£ +çݯ ä¿Ŀ +èĢĥ èĻij +æ·± åľ³ +严 éĩį +èĮĥ åĽ´ +å§Ķ åijĺ +çĽij 管 +ä¸ī 个 +è£ħ ä¿® +åħ¬ éĩĮ +åĪĨ åĪ« +çIJĨ 解 +éŁ © +åĬł å·¥ +认 羣 +ä¸į 好 +åİ» å¹´ +éĻį ä½İ +æľº ä¼ļ +åįı è®® +符 åIJĪ +å¢ŀ 强 +æĬĢ èĥ½ +é¦ĸ åħĪ +ç§ ¦ +ä¸ ģ +å° ¾ +æľī äºĨ +åľ° 产 +æ¸ ł +æĸ¹ 便 +移 åĬ¨ +éĢŁ 度 +å°¤ åħ¶ +éĢļ çŁ¥ +åĿ Ľ +éģ¿ åħį +æģ ¢ +è´ ¡ +èģĮ å·¥ +å®ŀ åĬĽ +æĺ¯ä¸Ģ ç§į +åIJ¯ åĬ¨ +çĸ¾ çĹħ +æĿ¥ äºĨ +缸 对 +çİ° å®ŀ +èŀį åIJĪ +åIJĮ æł· +åħ¬ åijĬ +çī¹ æ®Ĭ +ç´ « +ä¸ĭ åİ» +ä¼ł æĴŃ +æľĢ 好 +ä¼ĺ è´¨ +æ² Ĵ +æĮ º +æĹ ¦ +è¯ º +ä¸Ģ åIJį +éģĵ è·¯ +示 èĮĥ +è¿ĩ æĿ¥ +åIJĮ åѦ +é¼ ĵ +æĿ Ń +æľ¬ 次 +åIJĮ æĦı +ä¸ĸ 纪 +ç¾ Ĭ +æ¬ ² +å·¥ èīº +çĵ ¦ +人 士 +æľī æīĢ +ä»İ äºĭ +æľī å¾Īå¤ļ +ä¸į äºĨ +å²Ĺ ä½į +åıĺ å¾Ĺ +åĬ³ åĬ¨ +å¤Ħ äºİ +å¹³ åĿĩ +å½¢ 象 +å¡ ŀ +åħ± 享 +çĿ Ľ +åĪ© 润 +æŃ£ æĺ¯ +å¾Ģ å¾Ģ +缸 æ¯Ķ +æ¨ ª +åĪ · +æµĻ æ±Ł +大 éĥ¨åĪĨ +å¤ļ 个 +æĤ¨ çļĦ +çĶµ åķĨ +å¾® åįļ +å§ĭ ç»Ī +çĬ¯ 罪 +æĺ¯ åľ¨ +ç»Ħ åIJĪ +åİŁ æĿ¥ +æ¸ħ æ¥ļ +åIJĦ åľ° +æĦŁ åıĹ +å½ĵ ä¸Ń +è¶ĭ åĬ¿ +æĻ¯ åĮº +羣 æĺ¯ +ä¾Ľ åºĶ +转 åŀĭ +çĭ Ĥ +èĨ ľ +èĭ Ĺ +å¿ ł +å¾Ī 大 +èĤ¡ æĿĥ +ç¾İ åħĥ +æİĴ åIJį +åĬ¨ çī© +éĶ ħ +å¢ ¨ +主 å¸Ń +å¾Ī 好 +ç»Ŀ 对 +æĿ ľ +转 è½½ +çĴ ĥ +æĿij æ°ij +åIJ ¨ +åĽŃ åĮº +é«ĺ 度 +çī© è´¨ +è¾ ī +æĹ¥ 常 +æı Ĵ +ä¸ī å¹´ +ä½ĵ çİ° +æīį æĺ¯ +代 çIJĨ +ä¸į 管 +æģ Ĵ +åľ° ä½į +ç² ® +èĸ Ħ +æĺİ çĻ½ +ä¸Ģ èĩ´ +æĽ ¼ +åĵ Ń +åĩ ¤ +åĬ ² +æķ Į +æĪĺ æĸĹ +主 ä½ĵ +åħ¬ å¸ĥ +åıĤ èĢĥ +èĪª 空 +å¯ º +åѦ ä¼ļ +åıį æĺł +ç¾İ 丽 +太 éĺ³ +建 æĪIJ +æħ¢ æħ¢ +åIJĦ 个 +éĤ ¦ +ç»Ħ æĪIJ +ä¸ī 大 +éĶ ¦ +大å¤ļ æķ° +æ¦Ĥ 念 +éŃ Ĥ +åħ¬ çĽĬ +èį Ĵ +身 份 +æ·± åĪ» +åħ © +ç»ı åħ¸ +åIJĦ 项 +èĻ ķ +è¿Ľ æŃ¥ +åįģ äºĮ +æī§ æ³ķ +æĥ³ åĪ° +æĦŁ æŁĵ +åķĨ åĬ¡ +å°ı ç»Ħ +èĶ ¬ +çıŃ åŃIJ +åIJĮ å¿Ĺ +éĿ¢ 临 +çĤ Ĵ +å¤ļ ç§į +è§Ĥ çĤ¹ +åĵª éĩĮ +å° Ŀ +å§ Ĩ +èħ ¹ +åŁİ åĮº +太 å¤ļ +çĹħ æ¯Ĵ +åľ¨ äºİ +æīĢ è°ĵ +æĻ ° +æŀ Ŀ +æĭ ĸ +å® ħ +æķ´ æ²» +ä½ı æĪ¿ +åģ · +çĨ Ĭ +èµ ģ +æ° Ľ +æł¼ å±Ģ +åŁºç¡Ģ ä¸Ĭ +èĥ Ĩ +åħ ½ +鼶 åĶ® +åĿ ¡ +女 åŃ© +æĴ ŀ +åħ¨ åĬĽ +åĴ ĸ +èĤ © +çľ ī +èĩ³ äºİ +åħļ ç»Ħ +ä¸Ģ 件 +æĭ Ĩ +äºĭ å®ŀ +åĤ ³ +æ¹ ĺ +網 ç«Ļ +循 çݯ +åIJĮ æ¯Ķ +æĭ Ķ +åĮ» èį¯ +åħ» æ®ĸ +åĽº å®ļ +å®ŀéĻħ ä¸Ĭ +è®° å¾Ĺ +åĪ© äºİ +æĤ ¦ +æĭ ³ +èĤ Ŀ +æķĪ çĽĬ +è© ² +æ°ij 主 +çĹĩ çĬ¶ +é¢ ¨ +å¹¼ åĦ¿ +å§ ij +æĪ Ĵ +ä¸ĭ çļĦ +æ¸ ¡ +å¹´ åºķ +è®° å¿Ĩ +åIJ IJ +大 å¹ħ +å¾ ½ +åħ¬ ä¼Ĺ +ä¿¡ å¿ĥ +çİ Ľ +ä¼ļ ä¸Ĭ +ä¹ Ķ +æijĦ å½± +æ£ĭ çīĮ +éĻ ķ +åºĶ æĢ¥ +æĶ¶ è´¹ +æݧ èĤ¡ +仪 å¼ı +çŀ ¬ +æīĢ åľ¨ +ç¢ ° +å§ ĵ +é¡ Į +æĶ¯ éĥ¨ +使 åij½ +çĤ ī +å¯ Ħ +ç¿ ¼ +åľ° ä¸ĭ +è¾ ŀ +ä¿ ± +主 æĮģ +è´§ å¸ģ +æģ ¨ +èĤ Į +çĽ Ī +éĶ » +å¿Ĺ æĦ¿ +ç±» ä¼¼ +æĮ ĸ +éĢ » +ç¸ ½ +纪 念 +åķ ¥ +å¼ ¯ +åIJį åŃĹ +åģ¥ 身 +çļĦ å¿ĥ +é© ± +èĥĮ åIJİ +æ³ķ å¸Ī +ç² Ĵ +èĥ½ éĩı +è¾ ° +èī ³ +å½ ¼ +段 æŶéĹ´ +åIJĪ æ³ķ +æĵ ¦ +ç¾ ½ +åİ ¨ +æĪij 说 +äºĭ åĬ¡ +åĩł 天 +åħ ģ +ç¼ ´ +åį ĵ +两 ç§į +çĭ¬ çī¹ +å¸ ¶ +éĴ » +æĥ © +é¢Ĩ åħĪ +足 å¤Ł +å£ ³ +æĦıåij³ çĿĢ +åĪĨ å¸ĥ +ä¹ ĥ +éģ ĭ +ä½ © +è° ± +çģ £ +èį ¡ +è´¯ å½» +å¹ ¾ +ç£ ģ +åħ¸ åŀĭ +åī ĩ +åĨ » +æ¬ ł +ä¸į ä¹ħ +æµ ¦ +éŃ ħ +å¼Ģ äºĨ +使çĶ¨ èĢħ +è¿Ļ 款 +å° Ī +èĦ± è´« +æĶ» åĿļ +ç®Ĺ æĺ¯ +ç¨ Ģ +æĹł 人 +åł µ +å¥ ı +éĥ½ å¸Ĥ +åı¯ è§ģ +ä¸į åĩº +æ ·» +äº ı +ç¾İ 好 +èĥ ĸ +éŁ µ +æłĩ å¿Ĺ +èĬĤ èĥ½ +æĬ « +å° º +å¯ ¸ +ä¸Ģ 代 +é¢ Ĺ +èĢ ¶ +èĴ ¸ +åĸ ® +æ »¿ +çĮ ľ +æµ Ĩ +åŁ ĥ +åįĥ ä¸ĩ +èµ Į +èģ ² +ä½ľ é£İ +è³ ª +å¯ ¨ +å¹´ 人 +åį° 象 +æ¡ ¶ +æĴ ¤ +åįģ äºĶ +æ¯ ħ +æ² ª +åĽ½ æľī +大éĩı çļĦ +å¾ ¡ +å¯ ĵ +è¦ ĸ +æ¼Ĥ 亮 +çľ ł +ç ĤŃ +é» İ +èĻ ¹ +åĪ© äºļ +èŃ ī +æµ ı +åįģ åħ« +ä¸ ¢ +è¾ ½ +æľīä¸Ģ äºĽ +æħ Ī +åģľ 车 +å® ł +解 æĶ¾ +æľī å¤ļ +éĤ Ĭ +常 è§ģ +æĬ ¹ +çº ¤ +è¦ ª +æ¡ Ĩ +èİ ŀ +æ°§ åĮĸ +è¿Ļ 件 +åĩ ° +æŁ ´ +åıij çĶµ +é¼ ł +转 åĮĸ +å¨ ĥ +æĮ ¤ +ç½ © +å¯Ĩ åĪĩ +æĪij ä¸į +é«ĺ æĸ° +ä¸Ģ ç¯ĩ +è¿Ľ ç¨ĭ +è¡ ° +è¿ĺ ä¸į +ç ħĮ +æĸ° åįİ +èĤ ¿ +æ» © +ä¸Ģ æµģ +è¯ Ī +å®ŀ ä½ĵ +å¤ĸ åĽ½ +èº ² +èµ ł +è¦ º +æ¢ Ŀ +ä¸į è§ģ +è¨ Ĭ +åĮ ¹ +åį µ +çĩ ¥ +æħ ķ +é½ ¿ +å® ´ +é¥ ¼ +èij¡ èIJĦ +å°ı å¿ĥ +æģ ¼ +éĻ Į +æĺ Ĥ +åĥ ¹ +èĬ Ŀ +æ¯ı 个人 +åīį æıIJ +ä½ĵ ä¼ļ +æ¨ Ļ +æIJľ çĭIJ +对 åħ¶ +ä¸ § +èľ Ĥ +æµ ¸ +èª ¿ +åĿ ª +é¢ ĸ +åIJį 为 +ç¬ ¼ +èĪ Į +æľ¬ 书 +èģ ¯ +çº º +ç®Ģ 缴 +éĽ ¢ +ç¾İ çļĦ +éļ ¨ +é«ĺ å³° +è¿Ļ 家 +å Ĥ¬ +å° ¸ +ç¡ķ 士 +èŃ · +è° ¨ +æĺ ı +æĶ¿ åįı +è¡ Ķ +ç¿ Ĵ +åľ Ĵ +åĽ½ æ°ij +主 è§Ĵ +è£ ķ +ä¼ ª +åº ŀ +æ°ij èIJ¥ +æĥ § +ç§ĺ 书 +çĹ ķ +çĻ¾ åĪĨ +æº ¶ +æĹł çĸij +çļĦ çľ¼ +æĵ İ +ä¼Ł 大 +å½ ° +åħ¬å®ī å±Ģ +ç³ ķ +å¼ ¥ +åĤ Ļ +ä¹ ¾ +毫 ä¸į +注 æĺİ +åī¯ æĢ» +æĦ ī +æķ ¦ +é¦ ¨ +æĶ Ģ +éĢ Ŀ +åı¯ éĿł +å¤ ¸ +åľ ĺ +éĿ¢ ä¸Ĭ +æĬ ĸ +èĦ Ĩ +é© ° +ä¼ IJ +å¦ ¨ +å®ļ äºĨ +ç³ Ĭ +æŃ ¡ +éĥ¨ éķ¿ +ç§ ī +èĪ Ĩ +åĪij äºĭ +åIJ µ +æ¤ Ĵ +è¡ ĵ +è± « +èı © +åŃ µ +é¥ ² +å°± 好 +åł ª +ä¸ī è§Ĵ +åľº æ¯ĶèµĽ +ä¸į åģľ +æĵ ħ +åħ¨ æĸĩ +æ³ ģ +åѦ ä½į +æ± ° +éł ĺ +åı ł +éļ Ľ +å¸ IJ +çľĭ åĩº +åĮ ł +å±Ģ éĿ¢ +æ³ Į +è° Ĭ +åIJĮ æľŁ +æĬķ æłĩ +å¥ ´ +æĿ¥çľĭ çľĭ +èĦ ¾ +èŀ º +æŃ ī +çĽ ¯ +ç¨İ åĬ¡ +å» Ĭ +æİ © +æħ ¨ +çĽ ¼ +èĬ Ĵ +è® Ģ +æĮ £ +èĮ ħ +æĸ ¥ +æ¤ ħ +åĪ° æĿ¥ +èijĹ ä½ľ +çĭ ± +äºĮ æīĭ +ä»İ æĿ¥ +çĸ ² +åºĬ ä¸Ĭ +æĸ° 浪 +æ³ Ħ +å¢ŀ åĢ¼ +ä¸ Ľ +æļ ij +ä»İ ä¸ļ +æ· ĭ +å¤ļ æł· +æľ ´ +份 é¢Ŀ +æŀ £ +西 çľģ +æľ¬ è´¨ +æ·± æ·± +èī ĩ +ç» µ +产 åĢ¼ +æ¼ ł +èħ » +çŃ Ľ +åİ Į +æģ Ń +å«Į çĸij +æĪ ¶ +æ» ŀ +èĨ Ģ +åĬ £ +座 è°Ī +常 æĢģ +çļĦ æĥħ +è¦ ½ +å¯ Ĥ +åĮ Ĩ +èĩ º +é¡ ¯ +çķ ı +éģ £ +åį ľ +çŃī å¥ĸ +è² ¬ +æº ¯ +é İ +çĤ¹ 头 +èĵ ¬ +æ± º +éħ ¬ +éģ Ĭ +è³ ¼ +註 åĨĬ +æľ¬ æĬ¥ +çµ ķ +æ´» æĢ§ +åħ ij +éĮ ¯ +åĨ ¶ +åĸ » +æº ĸ +èĤ ¢ +æº ĥ +æĹ ¬ +åī Ĭ +çIJĨ äºĭ +å± ł +æ² § +èļ Ģ +鼻 åŃIJ +为 æŃ¢ +常 å§Ķ +çµ Ĥ +éĬ · +çĭ Ģ +ä¾ £ +èĥ Ģ +èŃ ° +çĶ¨ 车 +åĻ ª +æŃ · +åį Ķ +åĪ ¹ +ç«Ł æĺ¯ +é© Ĺ +èIJ Ŀ +çĻ « +çĹ « +æŃ § +å¼ Ĭ +åª ½ +çı Ĭ +è¡ · +éľ ī +åŁº çĿ£ +éļ ± +æ° ¨ +ç» ¸ +å°¼ æĸ¯ +çĥ ĺ +æľŁ åĨħ +è° ħ +éĽ ĩ +éļ Ļ +å ĸī +åī ¥ +çĹ ĺ +æĮ ½ +çĵ £ +æ¹ Ľ +æ¨ ± +æ¾ İ +æ¹ ĥ +åĨ¬ 奥 +æ£ µ +å® ° +åŀ Ĵ +æ§ ĭ +ä¾ Ī +èĮ Ħ +åĺ ¿ +èı ĩ +ç ĻĤ +åĬ ĥ +é į +èĶ ½ +çŀ Ń +æķ ŀ +ä¹ ĸ +éŁ § +è¾ ľ +æĩ Ī +ä½ £ +çŀ » +åŁ Ķ +èĪ ħ +å®ŀ äºĭ +é ¨ +å§ ¥ +çµ ¡ +åĺ » +çķ ¢ +æ²ĥ å°Ķ +è¿ Ħ +èĤ ĩ +æħ ij +ã § +ä ı +ð ł +ð¬ ĩ +ð« Ń +ð« IJ +ã ³ +© ½ +ð« ł +ã Ľ +ð¬ į +é ¿ +ð¬ Ĵ +ã Ļ +𬠤 +ð ¬´ +ð« ĸ +ð ¤ +ã ¬ +ä ² +ð« Ķ +ð« ļ +è¦ģ æ±Ĥ +ä¸Ģ äºĽ +å®ŀ çİ° +èĢĮ ä¸Ķ +åĽł æѤ +çĶ± äºİ +åħ³ äºİ +çĦ¶ åIJİ +æݨ åĬ¨ +ä¸Ģ æł· +æĮī çħ§ +è¿Ļæł· çļĦ +å½¢ æĪIJ +æľī äºĽ +æĽ´ åĬł +ç»ı è¿ĩ +建 è®® +æ²» çĸĹ +ä½ł 们 +æīį èĥ½ +ä¿ĥ è¿Ľ +åijĺ å·¥ +ä½ĵ éªĮ +èĪ ĩ +åģļ 好 +ä¿Ŀ è¯ģ +æķ´ 个 +æĺ¯ ä¸Ģ个 +éĩĩ çĶ¨ +çIJĨ 论 +æ¯Ķ å¦Ĥ +ä¸Ĭ çļĦ +æݨ èįIJ +çĶ³ 请 +天 空 +éĥ¨ èIJ½ +åįģ åĪĨ +æĿ¥ èĩª +ä¹ĭ éĹ´ +è°ĥ æķ´ +æ¯ı 天 +è°ĥ æŁ¥ +æĤ£ èĢħ +è¿ĩç¨ĭ ä¸Ń +é¦Ļ 港 +广 åijĬ +éĿ¢ 对 +满 足 +éķ¿ æľŁ +è§Ħ èĮĥ +æķ´ ä½ĵ +æĶ¹ åıĺ +æĻº æħ§ +å¦Ī å¦Ī +å¦Ĥ ä»Ĭ +åIJĪ åIJĮ +éĥ½ ä¼ļ +åĦ¿ ç«¥ +åĩı å°ij +éŁ³ ä¹IJ +ç»ı 常 +ä¸Ĭ å¸Ĥ +ä¼ĺ ç§Ģ +çļĦ éĩįè¦ģ +ä¸Ģ æĿ¡ +æµ· å¤ĸ +åı¦ å¤ĸ +ä¸Ģ 家 +åİĭ åĬĽ +大 åŀĭ +çľĭ çĿĢ +åĪ Ģ +幸 ç¦ı +æݨ 广 +åIJ Ľ +å¾ IJ +æī¾ åĪ° +äºİ æĺ¯ +èĩª 身 +ä¸Ģ ä½į +åľŁ åľ° +åĬł åħ¥ +æİ¢ ç´¢ +æ¢ ģ +主 åĬ¨ +å°± ä¸ļ +女 æĢ§ +çªģ çł´ +ä¸įåIJĮ çļĦ +è¿IJ è¾ĵ +èĩª çĶ± +å±ħ æ°ij +æѤ 次 +çļĦ æŶéĹ´ +家 éķ¿ +ä¸Ģ个 人 +æ£Ģ æµĭ +åĨħ éĥ¨ +广 å·ŀ +缴 æĴŃ +ä»İ èĢĮ +è´· 款 +åı¬ å¼Ģ +æĶ¹ éĢł +人 çĶŁ +å±ķ 示 +æ¯ı å¹´ +女 人 +çļĦ æĸ¹å¼ı +æķĪ çİĩ +å±± 举 +æ¸ł éģĵ +ä¼¼ ä¹İ +æ¡Ī 件 +åĪ© çĽĬ +çľĭ çľĭ +å¿ĥ éĩĮ +ç»´ æĬ¤ +å®Ŀ å®Ŀ +ç½ij ä¸Ĭ +论 åĿĽ +å°± åı¯ä»¥ +ä¸į 足 +æģ¢ å¤į +å¸ĥ å±Ģ +è´¡ çĮ® +ä¸ĭ éĻį +æİĮ æı¡ +çļ® èĤ¤ +å·¥ åħ· +éĩį åºĨ +åĵģ è´¨ +æݨ åĩº +çĶ· 人 +æī¿ æĭħ +çªģ åĩº +èĢĮ è¨Ģ +æ² Ł +åįı è°ĥ +æĺ¯ ä»Ģä¹Ī +æ± ¤ +æĴ ij +çĭ¬ ç«ĭ +çݯ èĬĤ +æī© 大 +æ´ ª +æĿ ° +çĽ IJ +ä» ģ +æ¶ī åıĬ +èĢģ 人 +åį³ 使 +åįĹ äº¬ +éħį åIJĪ +é¬ ¼ +çĪ¶ 亲 +ç½Ĺ æĸ¯ +å°ı åĮº +æķĻ æİĪ +åĨ³ çŃĸ +é¢Ħ 计 +æľ¬ 人 +ä¼ ¯ +ç« ¹ +åĪ° åºķ +å¸Ĥ æ°ij +åĩº åı£ +éĩĩ è´Ń +æĢ» ç»ĵ +æѦ æ±ī +åĬł 大 +广 举 +æµģ ç¨ĭ +人 åı£ +å¦Ĥæŀľ ä½ł +åĩº åİ» +åĩ ī +åĨľ æ°ij +çİ° 象 +åĬĽ 度 +ç»Ļ äºĪ +åħļ å§Ķ +è¯Ń è¨Ģ +线 ä¸Ĭ +æĢİ æł· +åĦ¿ åŃIJ +ç¡® å®ŀ +ä¹ĭ å¤ĸ +éĥ½ åľ¨ +èī ¾ +çļĦ æĥħåĨµ +éĩĮ çļĦ +åĽ´ ç»ķ +æĽ´å¤ļ çļĦ +ä¾Ŀ æ³ķ +åħ¬ åĽŃ +家 éĩĮ +æ¯į 亲 +ä¸į åĨį +èĭ ¹ +æ³ķ éĻ¢ +飩 åĽ½ +缸 å½ĵ +ä¸į çŁ¥ +è¯Ħ ä¼° +ä¸į çĶ¨ +顺 åĪ© +éĩį è§Ĩ +è´¢ åĬ¡ +ä»ĸ åĢij +åıij è¡Į +ä¸ĵ éŨ +åħ· å¤ĩ +并 ä¸įæĺ¯ +足 çIJĥ +é ŀĭ +åıij 表 +æ°¸ è¿ľ +èIJ¥ åħ» +éħį å¥Ĺ +æķ´ åIJĪ +è´ º +åĽŀ çŃĶ +æĶ¶ çĽĬ +ä¹Ł 许 +è» Ĭ +æİ¥ 触 +æĶ» åĩ» +åĽĽ å·Ŀ +æĢ§ èĥ½ +åĽŀ åĪ° +èħ ° +ä¹Ł 没æľī +å¼ Ħ +设 ç«ĭ +éĺ² æݧ +æĬĢ å·§ +éĢļ 常 +è´¢ æĶ¿ +éĥ¨ ç½² +åľº æĻ¯ +æ±Ł èĭı +表 è¾¾ +åĸ · +女 åĦ¿ +èĪ ¶ +çµ ¦ +ä¼ļ åijĺ +æĪĸ 许 +äº © +举 æĸ¹ +天 æ´¥ +è¿ij å¹´ +çľĭ æĿ¥ +æ¯Ķ ä¾ĭ +å² © +éĵ ľ +çİ » +å®ŀ éªĮ +æĢĿ ç»´ +æĭħ å¿ĥ +æ² Ī +身 è¾¹ +æ·± åĮĸ +ç²¾ åĩĨ +ç§ģ æľį +æ¶Ī éĺ² +åİ» äºĨ +ç»Ĩ èĥŀ +çIJĥ éĺŁ +æĺİ æĺŁ +é£Ł çī© +å¾Ī å¿« +让 ä½ł +ä¿¡ çĶ¨ +åĶ¯ ä¸Ģ +åħ¶ å®ĥ +çŃī æĸ¹éĿ¢ +å¾ĭ å¸Ī +æŃ» 亡 +æ Ł³ +ä¸Ģ æī¹ +ä¸Ĭ 涨 +æľº åľº +å½¢ åĬ¿ +æĦ¿ æĦı +éĽĨ ä½ĵ +æĸ° åŀĭ +æįŁ 失 +æĽ ¸ +ä¸ĭ åįĪ +æ¯ı 次 +æĪIJ å°± +åħ¬ è·¯ +èĻ « +åĴ ± +西 å®ī +æľĢ ä½³ +ç§ij çłĶ +å¤į æĿĤ +æľº åĻ¨ +çĪ± æĥħ +çħ§ çīĩ +å¹´ é¾Ħ +è³ĩ æĸĻ +ç² Ĺ +åĩĨ ç¡® +åĬł ä¸Ĭ +åĩº çīĪ +è° IJ +家 å±ħ +èĥĮ æĻ¯ +ä¸Ģ 线 +äºĭ 项 +åĬ¨ ä½ľ +ç¥ ¥ +æĢ» ä½ĵ +æĪ¿ åŃIJ +ä¹Ł å°±æĺ¯ +大 æ¦Ĥ +é«ĺ æķĪ +åIJ ¹ +æİ ĪæĿĥ +éĻĦ è¿ij +æ¡Ī ä¾ĭ +éĹ ¹ +çĪ¸ çĪ¸ +彩 票 +æĢ Ĵ +举 æĬ¥ +æĻ® éģį +çķĻ ä¸ĭ +è¡£ æľį +æĹłè®º æĺ¯ +åħħ 满 +æ·± 度 +æ¡ ij +æĪª èĩ³ +带æĿ¥ çļĦ +éĻ µ +æĦŁ æĥħ +èµ ļ +åĵª äºĽ +æķ´ æĶ¹ +æĪIJ çĨŁ +å¨ ľ +é¼ » +çŁ Ľ +çĽ ¾ +好 好 +第 åĽĽ +åĨł åĨĽ +è´¢ å¯Į +æľĢ 好çļĦ +车 åŀĭ +éĸ Ģ +åį³ å°Ĩ +åĪĨ 为 +éĿĴ å²Ľ +纷 纷 +ä»Ĭ æĹ¥ +å¹³ è¡¡ +å¹³æĸ¹ ç±³ +éĤ£ ç§į +åĩº çĶŁ +éĿĴ æĺ¥ +人 群 +人 å·¥ +ä¹ĭ ä¸ĭ +æ¹ĸ åĮĹ +åľ¨ æѤ +åįļ 士 +æŶ åĪ» +æ²³ åĮĹ +æĶ¾ å¼ĥ +éĢļ éģĵ +森 æŀĹ +çĸ Ĩ +æķ ¸ +èĬ ³ +æīĵ åĩ» +æĽ ¹ +åĮĸ åѦ +æĥ³ 象 +ä¸ĩ 人 +è´¢ ç»ı +åħĥ ç´ł +ä¼ļ 计 +åħ¨ ä½ĵ +æĦ Ľ +é«ĺ ä¸Ń +æľº éģĩ +声 éŁ³ +æĹħ è¡Į +æµ © +æŁ ± +å°ij å¹´ +åĽ½ å¤ĸ +èijĹ åIJį +çĶŁ åŃĺ +å§ ľ +带 é¢Ĩ +é¢ľ èī² +ä¸Ĭ ä¸ĭ +产ä¸ļ éĵ¾ +æĽ´ 好çļĦ +å² Ń +ä¼ĺ æĥł +便 æĺ¯ +åħ§ 容 +ä¸Ģ åıª +çIJ ´ +梦 æĥ³ +秣 èµģ +å¼Ģ åIJ¯ +è´Ń çī© +åĮħ åIJ« +åĪ© çİĩ +èµ· äºĨ +æľī åĬĽ +éĤ£ éĩĮ +审 æī¹ +对 æīĭ +çİ° éĩij +天 çĦ¶ +çĽ Ĵ +çĪ ½ +å¿ħ çĦ¶ +åĮĸ å·¥ +ä¸ĵ åĪ© +åķ ¡ +å¼Ģ å¿ĥ +人 ä½ĵ +éģĵ 士 +æĢģ 度 +空 è°ĥ +æĭĽ åķĨ +å§ » +第 äºĶ +æ£ Ĵ +ä¸Ģ ç³»åĪĹ +åį± æľº +转 åıĺ +åľº æīĢ +é¸ £ +æĪ¿ éĹ´ +éĢ ¼ +è¯ķ çĤ¹ +对 å¤ĸ +åĩº åı° +åľ¨ è¿Ļ +åİĤ 家 +å·¨ 大 +ç®Ģ ä»ĭ +çľĭ äºĨ +åħļ 建 +æĮĩ æĮ¥ +çŁ³ æ²¹ +ä¸į åı¯èĥ½ +èİ ² +ä¸į 太 +åĪĽ æĦı +第 ä¸Ģ个 +è´µ å·ŀ +è¿ĩ äºĨ +æľ¬ æĿ¥ +éģĵ å¾· +çŃĶ æ¡Ī +éĻ ¶ +ä¸Ģ è·¯ +èĤ ĸ +æ¸ħ æ´ģ +æľī æľº +åIJį åįķ +æĿ ± +åij¼ åIJ¸ +ä¸ Ī +ç¦ı 建 +è¯ķ éªĮ +å¼ķ åıij +ä¹Ł 没 +ä¸į ä½ı +çĨŁ æĤī +èIJ ¬ +ä¸į èī¯ +çł ĸ +èĩ´ åĬĽ +çŃ¾ 订 +åIJ Ĭ +ä¾ ¯ +çĺ ¦ +å§ij å¨ĺ +æĸ ¤ +妻 åŃIJ +æĺ¥ èĬĤ +çĪ ¬ +æĽ Ŀ +çĥŃ æĥħ +éķ¿ æ²Ļ +èIJ¥ éĢł +éħ · +éĵ Ŀ +åŁºæľ¬ ä¸Ĭ +åij¨ åĽ´ +ä»Ģ 麼 +认 åı¯ +åĪĨ åŃIJ +ä¸Ģ æĸ¹éĿ¢ +è½ ´ +å¼ · +马 ä¸Ĭ +éĽ ¾ +èĩ £ +å° ¿ +çĶŁ æĦı +å®ī å¾½ +ç¥ŀ ç»ı +åĩº å¸Ń +èį¯ åĵģ +çIJĨ çĶ± +åįı åIJĮ +æµģ åĬ¨ +åıij åĬ¨ +åĿļ å®ļ +表 æĺİ +åIJİ éĿ¢ +ä¹ī åĬ¡ +å¦ ĸ +æľī åı¯èĥ½ +å¹´è½» 人 +大 éĻĨ +å² ³ +ä¸į èµ· +çŀ¬ éĹ´ +ä¸įå¾Ĺ ä¸į +çŃ¾ 约 +åIJĪ æł¼ +åħļ æĶ¯éĥ¨ +æµİ åįĹ +便 åĪ© +éļı æŶ +å¥ ī +称 为 +产 æĿĥ +åIJ ķ +çĽ Ĩ +课 åłĤ +ç· ļ +æ£ ī +线 ä¸ĭ +èĩª è¡Į +举 æݪ +åݦ éŨ +èĩª ä¿¡ +å½± è§Ĩ +ä» Ķ +çĶŁæ´» ä¸Ń +æĿĥ çĽĬ +çĻ½ èī² +å°± ä¸į +è¿Ľ å±ķ +æ¯ı æĹ¥ +ä¾Ľ ç»Ļ +æĿĥ åĪ© +æĹł æķ° +çIJĨ è´¢ +ä¾Ŀ æŧ +ä¸Ĭ åįĪ +è¯Ĩ åĪ« +çĽĪ åĪ© +çł Ĥ +许 åı¯ +åIJĮ äºĭ +åĺ Ľ +éģ ¸ +çĿĢ åĬĽ +éŨ åı£ +ä¸į å¤ļ +åħ¶ 次 +ç¢ § +çī© çIJĨ +åĨħ å¿ĥ +çĻ¾ å§ĵ +æĢ» 绣 +å¹² åĩĢ +积 ç´¯ +åıį é¦Ī +æłij ç«ĭ +社 交 +ç§ © +åįģ ä¸Ģ +éĤ ĵ +驱 åĬ¨ +å±ķ è§Ī +èĪĴ éĢĤ +åŁº åĽł +å·® å¼Ĥ +转 让 +å°ı å§IJ +æł· åŃIJ +ç¿ Ķ +é«ĺ åħ´ +å½±åĵį åĬĽ +æīĭ ç»Ń +缸 åIJĮ +缸 åºĶ +æĻ Ĵ +è§ Ģ +å¸Ĥ å§Ķ +èĬ ¯ +å±ķ çİ° +åľ° çIJĥ +éĤ ª +ä¸Ģå®ļ çļĦ +åħģ 许 +ä¿¡ ä»» +æī ij +éĻ¢ æł¡ +ç®Ģ 称 +åģļ æ³ķ +ä¹ĭ è·¯ +æĹĹ ä¸ĭ +èħ Ķ +æ¶Ī 失 +ä¸ĸçķĮ ä¸Ĭ +åŁİ 乡 +èĪŀ åı° +å¾Ī 大çļĦ +绣 çŃ¹ +åħ¬ å¹³ +èĤ ¾ +çļĦ 好 +æ± ģ +çľ¼ åīį +éĽ £ +å¹ ½ +åħ± 产 +主 åĬŀ +å¤Ħ ç½ļ +åº Ļ +éģĵ çIJĨ +å¼ µ +æİ¥ çĿĢ +çĮ İ +çģ Į +çĶ± æѤ +人 åĬĽ +æµģ è¡Į +ä¾ ł +åı¯ä»¥ 说 +èĴ ĭ +å½¢ æĢģ +æĹ¥ åŃIJ +æ¼ Ĩ +çķĻ åѦ +缸 éĹľ +æľĢ å¤ļ +åĩŃ åĢŁ +åħ¬ 交 +æĮĸ æİĺ +æĿĤ å¿Ĺ +主 人 +éļľ ç¢į +æł¡ éķ¿ +æĸ¹ ä½į +ä¸Ĭ çıŃ +å¤ļ åħĥ +è ĥģ +éŃħ åĬĽ +èĮ Ĥ +åħħ çĶµ +强 大 +çĥ ¤ +å¥ĭ æĸĹ +å®ŀ çĶ¨ +éĺ ģ +ç»Ļ äºĨ +æľ¬ ç§ij +æł ĭ +æĭ ¨ +æķĻ ç»ĥ +éĥ½ çŁ¥éģĵ +æ¯ķä¸ļ çĶŁ +ç¢ Ĺ +åŀ Ĥ +è® ¼ +å®ģ æ³¢ +åѦ èĢħ +è°¢ è°¢ +åŁİ éķĩ +æĢİä¹Ī åĬŀ +éģ Ķ +æĪIJ 交 +æ½ľ åĬĽ +åį § +æĸ° å¼Ģ +éħį å¤ĩ +主 åĬĽ +åij³ éģĵ +çĥ Ĥ +é£ŀ è¡Į +å« ģ +大 大 +ç»Ļ 大家 +å¤ĸ éĿ¢ +éĨ ī +åıij è¨Ģ +æĹ© é¤IJ +åIJĦ èĩª +å® Ļ +èį£ èªī +æĬ« éľ² +é¡ ŀ +åĨħ çļĦ +èĤ ª +è¾ IJ +æ³ µ +æĬ Ľ +æĺŁ æľŁ +ä¸Ģ 带 +çĶŁ ç´ł +ç»ı éĶĢ +åĩ ¶ +åľ° ä¸Ĭ +åij½ è¿IJ +åĵ ² +ä¸Ĭ åİ» +æĸĩ çī© +è¯ ij +æĮ¯ åħ´ +éķ¿ æŶéĹ´ +ç¥ Ń +åIJĪ èĤ¥ +è¿Ŀ è§Ħ +èģ ª +ä½İ äºİ +éĢĤ å½ĵ +æľī åºı +æľ¬ ç½ij +çķĻ è¨Ģ +æĥ³ æ³ķ +çŃ¾ ç½² +å§ ļ +æĢ§ æł¼ +èĴĻ åı¤ +æŁ ı +åŀ « +åѦ åİĨ +ä»ħ ä»ħ +讲 è¯Ŀ +éĶ IJ +æĢ ĸ +åī ª +èĭ į +åIJ ĵ +强 çĥĪ +åģ¥ åħ¨ +çĸ ¯ +åı¤ 代 +å¥ Ī +ä¸į çĦ¶ +乡 éķĩ +æľĭåıĭ 们 +åĤ ħ +èģ ½ +个 æĢ§ +æ³ķ è§Ħ +å°ı éķĩ +çĶ» éĿ¢ +第 åħŃ +網 è·¯ +åīį æĻ¯ +åIJ¬ 说 +ä¼ł åªĴ +æĿ¡ ä¾ĭ +åĪ« çļĦ +ä¸į æĩĤ +顾 éĹ® +强 度 +éĺ¿ éĩĮ +èµ° åĬ¿ +å¸ ½ +çļĦ ç¡® +åĮº åĪ« +éĮ ¢ +主 管 +ä¸Ģ çľĭ +æĸ ľ +åŃĺåľ¨ çļĦ +ä» ² +åį± 害 +éĵ Ń +游æĪı ä¸Ń +éħ ± +é¾Ļ 头 +人 å¿ĥ +éĢĢ ä¼ij +æµı è§Ī +åĬ « +éĺ² æ²» +ç® Ń +å± Ī +è¾½ å®ģ +å£ ¤ +è¿İ æĿ¥ +éŀ į +çĶ¨ æĿ¥ +大 åľ° +ä» ° +éĢļ 讯 +å¼Ģ å·¥ +è£ ¤ +å¦Ĥ åIJĮ +éª ¤ +éĺŁ åijĺ +è½ © +ç¾İ æľ¯ +èĻ Ł +åIJĮ ä¸Ģ +åľ ĸ +书 æ³ķ +æīĵ åį° +åIJ« æľī +éĽĨ æĪIJ +éĹ · +å¸Ĥåľº ä¸Ĭ +æĹģ è¾¹ +åľ° æĿ¿ +产çĶŁ çļĦ +ç² ¤ +éĩį ç»Ħ +è¡Ģ 液 +çŃ ĭ +åĬŀ äºĭ +常è§ģ çļĦ +ä¸Ĭ åįĬå¹´ +å±ı å¹ķ +åIJī æŀĹ +å· © +åĸľ çĪ± +ç¿ ł +ä¸ī ç§į +æ¡Ĩ æŀ¶ +举 èİŀ +çĶĺ èĤĥ +èĬ ¬ +åĽ¾ 书 +åĩ¤ åĩ° +æ°Ķ åĢĻ +å° ´ +å° ¬ +两 天 +è¾ħ 导 +åĢŁ 款 +æĹ¥ èµ· +æ´ Ĵ +ä¸Ģ 度 +è¹ Ī +æ½ Ń +æī ĩ +çĻ ľ +æĸ° åħ´ +åĤ ² +诸 å¤ļ +è´ ª +éĻ· åħ¥ +èĪ Ł +èĤº çĤİ +ä¸Ģ æł·çļĦ +åİ ĺ +åľ° çIJĨ +æĬķ 注 +éļ Ĭ +åħī ä¼ı +ä¿Ŀ åģ¥ +åħ Ķ +åħ¬ åĬ¡ +æīĵ çł´ +çĶ· åŃ© +åĬ³ åĬ¡ +ä½ł ä¼ļ +çĶ¨ åľ° +æº ¢ +åıij è¾¾ +èĤ ļ +è¿ĩ äºİ +èĩ Ĥ +éĢĻ 樣 +è½» è½» +ä¸Ń åħ± +åIJĦ åĽ½ +åĶ ĩ +å®ŀ ä¹ł +èĻ ¾ +æ§ ½ +ä¸į ä¸Ĭ +åħį çĸ« +åįł æį® +å·¥ ä¼ļ +åĽ Ĭ +èĪª 天 +åı¯ çĪ± +æĸĹ äºī +çĺ ¤ +å¦Ĥ æľī +éĽ ĸ +对 æĪij +åĩº 秣 +好 çľĭ +太 大 +æ°´ åĪ© +åĬ¿ åĬĽ +åħ¨ æ°ij +ç½ ¢ +èµ¢ å¾Ĺ +çĶµ ä¿¡ +车 éĹ´ +æĻĤ åĢĻ +å°ij æķ° +éĵ ¸ +åħ³ èģĶ +ä¸įä»ħ ä»ħ +为 æĤ¨ +åĴ ¸ +æľº åĬ¨ +è£ Ļ +åĵį åºĶ +éģ ł +è² · +ç© ´ +å¢ ħ +éĶ ¡ +çµ Ħ +çģ« 车 +è³ĩ è¨Ĭ +åĨ³ èµĽ +污 æ°´ +èª ŀ +å´ Ľ +ç´§ å¯Ĩ +缺 å°ij +å¤ļ 人 +æĢ» 书记 +éĶ Ī +èij Ľ +å¿ĺ è®° +éĻĮ çĶŁ +éķ¿ 大 +åħĪè¿Ľ çļĦ +ç¡ ħ +åıij æĺİ +å©´ åĦ¿ +æīİ å®ŀ +èĽĭ çĻ½ +ä¸Ģ çĻ¾ +缮 åħī +æ ħĮ +åĬł æ²¹ +åIJ ŀ +ä¸Ģ 群 +ä¸Ń ä»ĭ +å¸ ĸ +å¿ Į +èģĮ èĥ½ +广 æĴŃ +çĽij å¯Ł +ç§ĺ å¯Ĩ +çĭ ® +è¿Ļ æĿ¡ +éĢ ¢ +æĢ ¨ +åįģ åħŃ +è© ¦ +说 åĪ° +åĩĿ èģļ +æĮĩ 示 +æ° ¢ +å¼ ĺ +éĺ Ģ +æĸ © +éł ħ +ä¸Ģ å¼Ģå§ĭ +æİĴ è¡Į +åľ¨ æĪij +纪 å½ķ +æĬ Ħ +æł ª +说 æ³ķ +ä¸Ń èį¯ +好 å¤ļ +åıª ä¸įè¿ĩ +çķĻ åľ¨ +个 å°ıæŶ +认 çŁ¥ +çķ « +è§ģ è¿ĩ +å°ı å¾® +ä½Ľ å±± +çľ ¾ +讲 è¿° +æ¢ ³ +称 åı· +æĹ¥ æĻļ +è¢ ĸ +åķ ¤ +æľª ç»ı +æľĢ æĹ© +æī® æ¼Ķ +è¡Ģ 管 +çº ± +æĥħ èĬĤ +第 ä¸ĥ +æį § +ä» Ĺ +æ¿Ģ çĥĪ +æĹł 线 +ä¸į 容æĺĵ +å¼Ģ å¹ķ +æĸ° çĶŁ +ä¸ĵ 注 +èij ± +åįĹ æµ· +çĩ Ł +èµ· ä¾Ĩ +æ´¾ åĩº +åĦ Ĵ +ä¾ ¨ +è¼ ĥ +åįļ è§Ī +éĢ ¾ +åĮ Ģ +ç»ıæµİ åѦ +æ¸ Ĺ +ä¿Ŀ èŃ· +çī º +çī ² +çİ « +çij ° +æľĢåIJİ ä¸Ģ +æĶ¿ åĬ¡ +æ§ Ľ +èĻķ çIJĨ +éļIJ æĤ£ +æī¿ åĮħ +æ¥ µ +æ¡ © +çĽ ² +导 åIJij +èĩ´ å¯Į +ç¼ Ĩ +æģĭ çĪ± +ä¸į åĬ¨ +ç»Ļ 人 +å· ¢ +表 æĥħ +举 åįĹ +åĨħ å¤ĸ +è¾Ī åŃIJ +åı ī +åįļ ä¼ļ +åĬŁ æķĪ +æ¸ ´ +å± ¬ +æİĴ éĻ¤ +éĢ Ľ +ä¸Ģ ä¼ļ +ä¸į å¼Ģ +å¼Ģ å¥ĸ +é»ij é¾Ļ +é»ijé¾Ļ æ±Ł +å¿« ä¸ī +度 åģĩ +åĿ ¤ +éĤ® 件 +æĩ Ĵ +ä¾Ľ çĶµ +å» £ +好 è¯Ħ +ç§ĺ书 éķ¿ +æĪĺ åľº +好 å¥ĩ +ä¾µ æĿĥ +æĨ ¾ +æľĢ åĪĿ +æī¹ åıij +åİ ķ +è¼ ķ +æŀ ¯ +ä¸ļ åĨħ +è´Ń æĪ¿ +ä¸į åľ¨ +纪 å§Ķ +æīĢ éľĢ +å¸Ĥ éķ¿ +è³ ½ +å¼ķ æĵİ +çģµ éŃĤ +éĬ Ģ +æ» ¤ +çĿ IJ +å¤ļ 项 +åĽŀ 头 +èī ĺ +å¤į å·¥ +éĥ¨ 件 +ç´§ ç´§ +æŁIJ ç§į +使 åħ¶ +æĸ° 人 +æŀ ļ +æ³ķ å®ļ +å·´ å·´ +涵 çĽĸ +ç¨ » +æĭ ¾ +æĻ ķ +è½ ¿ +éĢļ è¡Į +åĵ Ģ +æ³ Ĭ +温 馨 +éĽĨ èģļ +çĨ Ļ +åĩ ij +åįģ ä¸ĥ +æ°Ķ æģ¯ +æıIJä¾Ľ çļĦ +æ³ ³ +奥 è¿IJ +çģ¾ 害 +åĩĢ åĮĸ +è·¨ è¶Ĭ +åĵª æĢķ +éŁ ¿ +å¢ŀ æ·» +çĦ Ĭ +æ®ĭ çĸ¾ +ç¢ Į +æĤ Ķ +è§ģ è¯ģ +è¾ĸ åĮº +å¿ĥ èĦı +éļ § +åį ¸ +åı¯èĥ½ æĢ§ +æľī 趣 +åī¯ 书记 +åĮĸ å¦Ĩ +ä¿ Ĥ +æ£ ļ +éĨ ĩ +带 头 +éł Ī +追 究 +æij Ķ +è¿Ļ éĥ¨ +ä¸į 论 +ç¥ ¸ +å ³» +éģ ķ +çĶŁ èĤ² +å¤ ł +å¤ĸ 交 +è¯Ħ 为 +ä»İ å°ı +å°ı å°ı +é ¥¿ +æĴ ¼ +è·¨ å¢ĥ +被 åijĬ +åįĹ å®ģ +身 å¿ĥ +åĨį çĶŁ +æīĢ 说 +æŶéĹ´ åĨħ +åĪĹ åħ¥ +éĿĴ æµ· +çĪ± 好 +çª Ħ +èĪ Ī +è¿ĩ 渡 +æ¿ Ł +éĽ Ģ +审 è®® +åĽ½ èµĦ +æŃ¥ ä¼IJ +轨 éģĵ +ä¿¡ 念 +ä¸ī åĪĨ +çĨ ¬ +åѵ åĮĸ +ç¼ ł +éĥ Ĭ +èĪĴ æľį +纪 æ£Ģ +ä¸Ģä¸ĭ åŃIJ +鼻 話 +è² ł +éĴ ¥ +åĮ Ļ +çĹ ´ +è¶ ģ +ç» £ +çĪ µ +è½ ° +éª Ħ +å§ ¨ +æĭ ĺ +çĮ ´ +è® ¶ +è¿Ļ 座 +çį ¨ +æ·ĺ æ±° +çĹħ ä¾ĭ +æ²Ļ åıij +è§Ĩ 为 +头 æĿ¡ +å¿ħè¦ģ çļĦ +åı¯ è°ĵ +è¯Ŀ 说 +ç¯ Ħ +æĹ© çĤ¹ +æŀ¢ 纽 +ç¾ ¡ +çĪ± åĽ½ +çªģ åıij +éĢ Ĭ +æ½ į +èį£ èĢĢ +èŁ ¹ +æ¦Ĥ çİĩ +å¾Ī ä¹ħ +æĥ ķ +è¨ ´ +åľĨ 满 +çļ ± +åĪĨ æ³Į +åħħ 足 +çľĭ æ³ķ +è¾ Ł +æĭ ¦ +æĭ © +对 åºĶ +为 æł¸å¿ĥ +èħ Ĭ +å¤ļ ä¹Ī +æµ ij +å®ı è§Ĥ +èĦ ĸ +åIJĪ èµĦ +çĶŁ 涯 +å®ŀ è´¨ +ä¼ĺ çĤ¹ +çĶ¨ æ°´ +寿 åij½ +æ² « +åIJ ģ +è© ¹ +åĽ½ éĺ² +å´ © +åĿ İ +èĨ ı +ä¸Ģ è½® +éģĹ äº§ +æ¹¾ åĮº +ç» İ +åįķ 纯 +æ¾ Ħ +åīį åĪĹ +身 å½± +é»ĺ é»ĺ +æį ī +çĴ ° +èı Ĭ +æĢ ľ +åħĭ æĢĿ +æĢ» å±Ģ +çĩĥ æĸĻ +ä¸ļ æĢģ +åIJĦ æł· +åĴ ½ +åĩº èī² +åĪĿ å¿ĥ +åı Ľ +çłĶ 讨 +è¡ « +åİĨ ç¨ĭ +ç¦ ½ +è¶³å¤Ł çļĦ +èį Ĩ +çľĭ å¾ħ +è´ © +åĨ³ å¿ĥ +è£ ¹ +å¸Ī èĮĥ +åŀ Ħ +æĿ ł +åĩ ¸ +çĬ¹ 豫 +çĥŃ è¡Ģ +åIJĪ ä¼Ļ +éħ µ +èIJ½ åľ¨ +åįł åľ° +è¡ ¬ +èĵ ī +æĦ ¤ +æ¸ Ĭ +åĪĨ æķ° +ç¬ij çĿĢ +太 å¹³ +çĤ « +æݨ ä»ĭ +æĸ¯ åĿ¦ +å½¢ 容 +æĵ Ĭ +æĦŁ åħ´è¶£ +åĨĽ 人 +åĩĮ æĻ¨ +对 çħ§ +åıij çĹħ +å· ¾ +èĪ ī +æª ¢ +ç¬ij äºĨ +ç¡® è¯Ĭ +è´Ł åĢº +壮 大 +æĪ ļ +äºĴ èģĶ +èª ² +èħ ¦ +æĹ ± +åıĹ æ¬¢è¿İ +åį ī +éĻ¢ 士 +æ© ¡ +ä¸Ģ 对 +è¾ ± +æ² Ĥ +åı² ä¸Ĭ +æIJ ı +å´ ĸ +代 è°¢ +ç£ · +é¡ ĺ +æµ ĩ +常 çĶ¨ +åį ij +åĩº åĽ½ +è¯ ł +稳 æŃ¥ +ç»ı 纪 +å¤ļ å¤ļ +æīĢ å¾Ĺ +为 主é¢ĺ +ä¸Ģ åĪĨ +æł ½ +é¡ § +çº ² +åĥ ħ +å£ ĵ +åĦ ª +ç¿ ° +æİ Ģ +人 为 +åª ³ +æ´ ½ +èĿ ¶ +å¤į åħ´ +ä¼ļ å½±åĵį +åIJĦ çķĮ +éĤ£ ä¸Ģ +é¢ ¤ +çĢ ı +çĢı 覽 +å¯ ŀ +åı¯ æĢķ +åį³ æŶ +çķ ´ +ä¸ĭ åįĬå¹´ +ç¬Ķ è®° +éĻĦ åĬł +çĥŃ æ°´ +å¥ ¸ +ç£ ħ +æĿ ī +æ¸ħ åįİ +éĸ ± +ç° ¡ +å¤Ħ å¤Ħ +åIJĪ éĩij +æ²³ æµģ +ç´ ° +è´Ł éĿ¢ +çļĦ 羣å®ŀ +åĻ¨ 械 +èĴ IJ +西 äºļ +å· ħ +ç² ¹ +åİŁ æĸĩ +æŀ ķ +è¡Ģ åİĭ +åļ ´ +å¸ ĺ +åĨ Ģ +æĮ « +çĶµ è·¯ +å°ı ä¼Ļä¼´ +èĿ ´ +æľĢ å¿« +æĭ Į +å® ª +æĸ · +ç¿ ħ +åĴ ³ +åĹ ½ +ç¾ ŀ +躺 åľ¨ +èµĽ 车 +æ² IJ +éĻIJ 度 +为 ä¸Ģä½ĵ +èĴ ľ +å¹ « +æIJ ħ +åĭ ĭ +åī ĸ +纳 ç¨İ +éķ¿ æķĪ +ç½ ķ +åī¯ æľ¬ +ç© į +éĴ © +ç¹ ¼ +åĽ½ åľŁ +è¼ ī +ä¸į å¿ĺ +èѦ 示 +çģ ¿ +å¿ĥ å¾Ĺ +æĦ ļ +忽 çķ¥ +åĽŀ äºĭ +åįł æľī +æ· Ħ +çī ¡ +çĽij äºĭ +ç¿ ¡ +éĴĪ对 æĢ§ +çª ĥ +è£ ½ +èĨ Ŀ +ç³ Ł +港 æ¾³ +太 太 +æ¾ ¡ +ç»Ĩ åĮĸ +åĶ® åIJİ +å®ŀåľ¨ æĺ¯ +ç« £ +çį ² +åĢ¾ åIJij +å¼ķ çĶ¨ +é¹ ħ +ç¬ij 容 +ä¹IJ 趣 +æ°ij æĶ¿ +éŨ æĪ· +å± ģ +è¿· 失 +éĶ Į +å°ı 康 +åĭ ī +æ³ ¼ +ä¾ĭ åŃIJ +ä¸ī ä½į +å» ł +èĶ ĵ +广 éĺĶ +èĢ į +èĢģ èĻİ +åĭŁ éĽĨ +èĦļ æŃ¥ +æĭ ¯ +åŃĹ åı· +çĦ ° +é¢ ł +èļ Ĥ +èļ ģ +é£ ¯ +人 æĢ§ +æĴ ° +åİ ¢ +å±Ģ éĻIJ +æľª æĪIJ +åĵª åĦ¿ +大 åıij +ä¸į å®ļ +å¾ģ æ±Ĥ +éĥ µ +åĢº æĿĥ +çĪ± ä½ł +èº ģ +ä»ħ ä¾Ľ +è¿ľ å¤Ħ +éĨ Ľ +åĥ µ +积æŀģ æĢ§ +æİ ¡ +åīį ä¸ī +äºİ ä¸Ģä½ĵ +çŀ Ħ +çĿ ģ +æ² ¸ +åħ± èµ¢ +éĢĢ å½¹ +è´Ŀ å°Ķ +æİ ı +æĪ ² +è¡ į +éĶ Ĥ +ä¸ĩ ä½Ļ +ç§ij åĪĽ +æ¼Ķ åĶ± +欧 åħĥ +æ·¡ æ·¡ +éĿĴ å±± +èĹ Ŀ +ç» ½ +令 çīĮ +éĽĨ 群 +ä½ľ çī© +çĢ ij +å¤ ¯ +ç½ij 游 +åħ« 大 +éª ļ +èª ĵ +ä¼ļ å±ķ +åħļ åı² +æ£Ģå¯Ł éĻ¢ +åĸ ĺ +éĺ ± +èĢĮ åĩº +éĢļ 车 +éĴ ĵ +æĥħ 人 +æ¸ Ľ +ä¸Ń ç§ĭ +çĪ Ń +åıª åī© +æĺ Ķ +éĩİ çĶŁ +ç¡ « +èIJĿ åįľ +æĬµ æĬĹ +çĻ« çĹ« +éĻ Ģ +èĶ ļ +å¸ ľ +满 满 +èı ± +éļĨ éĩį +æĺŁ çº§ +æ½ ĩ +åħ¬ åħĥ +è° £ +æ¯Ķ äºļ +æ¡Į åŃIJ +èµ £ +è² ¼ +æĦ¿ æľĽ +é¡ ½ +æ´¾ éģ£ +ç¥ Ľ +åª ļ +éĺ ľ +èij « +èĬ ¦ +æ³ » +å¡ Į +çĭ Ń +å»ī æĶ¿ +å¥ij æľº +æĹĹ èĪ° +æĥ « +严 åİī +åıĭ æĥħ +å¦ Ĭ +å¨ ł +åĵª 家 +èĨ ¨ +è¶ Ł +æĮ ª +èĻ IJ +é łģ +çŀ © +éº Ł +ç¨ £ +èģĶ éĢļ +åı ® +çİĭ èĢħ +ä¸į ç¡®å®ļ +ç ijľ +è° İ +çī¢ è®° +ç¢ ¼ +æĬ¤ èĤ¤ +é¡ · +çĦ ķ +åģļ 强 +éļ± ç§ģ +éļ±ç§ģ æ¬Ĭ +åıĹ å®³ +ä¸į çĶ± +çĥ ¹ +é¥ ª +é© ³ +ä¼ ½ +ä¸Ŀ 绸 +è¥ Ħ +åįģ ä½Ļ +éº Ĺ +æ¬Ĭ åĪ© +èģ ŀ +åı¤ èĢģ +éģ ı +åIJĦ å¼ı +å°± è¡Į +åħ¥ å¢ĥ +ç ĥģ +èľ ĺ +èĽ Ľ +çº ¬ +çŁ « +è» Ł +æ´Ĺ è¡£ +æĦ § +é¢Ħ æ¡Ī +éľ Ĩ +æ·± åİļ +éĺ¿ æĭī +åĨĻ åŃĹ +åį ¦ +éķ Ģ +模 æł· +åĤ į +æIJ į +èĸ ¯ +åł ħ +åħ¬ 积 +è¨ İ +ä¼ł æŁĵ +æ¯ ¯ +çIJĨ å·¥ +åĨ· éĵ¾ +ç«ĭ æĸ¹ +æ¢ Ń +åľ£ è¯ŀ +综 èīº +çİ© ç¬ij +æĥ³ ä¸įåĪ° +æijĩ 头 +æ· ¹ +åģĩ æĹ¥ +åĢ ĺ +èĢ ½ +èİ ĵ +åŁ · +èĩª è´¸ +åįĬ 天 +æª Ķ +æ¾İ æ¹ĥ +éķ ij +ä¸ « +éĩĮ ç¨ĭ +å¼Ģ èįĴ +èı ı +å®Ŀ è´µ +èŃ ¬ +åķ Ł +æŁ ł +æª ¬ +é© Ń +æ± Ľ +çĨĬ çĮ« +èķ ī +éļı ä¹ĭ +å± ij +è¾ĥ 强 +èĥ ³ +èĨ Ĭ +éĿĻ éĿĻ +åĴ ª +æĭĽ åij¼ +代 è¨Ģ +ä¿¡ ç®± +è£ħ éħį +æĤ į +åįķ 车 +èIJ İ +å¤ļ 彩 +éĻ ¸ +ä»İ 严 +æ© Ħ +æ¦ Ħ +éĢ ® +éĩĮ æĸ¯ +姿 æĢģ +太 æŀģ +éĩ Ŀ +æº ī +è¿ Ń +ç§ ¸ +ç§ Ĩ +å·¥ å§Ķ +æ± ķ +èģ Ĩ +ä½ ¬ +ç¼ ħ +çĶ ¸ +åī¯ å±Ģéķ¿ +éĹ º +èª ¤ +è¤ IJ +ä¸į éĻIJ +èħ ķ +åij ķ +çŁ ¶ +åĨľ 家 +管 å§Ķä¼ļ +é¥ º +èĬ ľ +æ¾ Ī +è© ¢ +å¨ģ å°¼æĸ¯ +ä½ķ åĨµ +å°ı ä¼Ļ +奢 ä¾Ī +è¿Ļ ç¯ĩ +è¯ µ +ç«ł ç¨ĭ +ç´ Ģ +éIJ ĺ +éĤ ¢ +ç³ Ļ +ç¼ Ģ +ä¹ Ĵ +ä¹ ĵ +çī¢ åĽº +åĿ ŀ +å¼ Ī +ä¾ĭ å¤ĸ +å» ³ +è§Ħ ç«ł +èĬ Ļ +ç¯ · +èº ¯ +æł Ī +åĿļ å®ŀ +åŁº 建 +çĿĢ çľ¼ +ç· ´ +èij © +ç¼ ļ +æ¦ Ĩ +主 åĭķ +ç¥ Ģ +äºĴ éĢļ +å°¤ 为 +å® Ľ +éª ¼ +æ± ² +ä¾ ĥ +æĤł ä¹ħ +æij § +æĭ ĩ +é« ĵ +éº Ĵ +éĻ Ľ +æŀ ¸ +æĿ ŀ +è´ ¬ +å°ı é¾Ļ +åĵ ® +èĵ¬ åĭĥ +åĮ Ī +çķľ çī§ +å¨ © +个 å¤ļ +æ² ¥ +æĺ § +çĦ ļ +æĬij éĥģ +çĸ ¡ +èĺ ij +éģİ ç¨ĭ +æ© ± +éĿ ĵ +大 çIJĨ +é« ¦ +åĪĨ 辨 +æ¸ ¤ +çĸ ¤ +åĬ¨ èĥ½ +å¼ł 家 +ä¸ĩ åįĥ +æ» ¥ +é¥ ¥ +åºŁ å¼ĥ +å¸ ³ +æ¼ ³ +è± IJ +ä» ij +å« ī +å¦ Ĵ +çŀ Ĵ +è¡ ħ +çĭ ¸ +å¾ģ ç¨ĭ +éĤ ¯ +éĥ ¸ +ç¥ Ī +ç¥ · +è¶ ´ +ç»ĵæŀĦ æĢ§ +è§Ĩ åIJ¬ +è¬ Ŀ +çĴ Ģ +çĴ ¨ +åĩº å¤Ħ +è¯ Ģ +å¾ ĺ +å¾ Ĭ +çľ ¨ +åĸ ĩ +åı Ń +åĺ ² +çķ ¸ +å¹² äºĭ +æļ § +æ² Ľ +åĦ Ħ +å» ĵ +åİ¿ éķ¿ +èĥ ļ +çIJ ¢ +çŃ · +éĩ ĭ +ä¾ ® +åIJ © +åĴ IJ +åĮ ¿ +æĬ¬ èµ· +æ³ £ +æ¶ ¤ +éº ½ +æĽ Ļ +åī¯ éĻ¢éķ¿ +åħļ åĴĮ +æķ£ åıij +润 æ»ij +åĵ º +æĥ ¬ +漫 éķ¿ +ä¸į æĩĪ +åŁ ł +åĹ ĵ +èĢģ çĪ· +è® ½ +æĪĺ ç»ĦåIJĪ +æ£ ł +åħ¨ åŁŁ +èł ¢ +è¯ ¡ +åīį çŀ» +æķ Ľ +ä¸Ģ å°ģ +å¹ Ĥ +èİ Ĩ +è¯Ŀ è¯Ń +ç»Ĩ åĪĻ +å± ¿ +åµ Į +éĢ į +åĺ ± +æ¸ ² +çĥ ¯ +çĿ ¹ +é¦ Ĵ +èħ ¥ +æĬĹ åĩ» +çĿ « +èį Ķ +éļ İ +æ³ī æ°´ +è¬ Ĥ +ç Ĥ¬ +åĩı æİĴ +è¸ Ĭ +è ·» +æ· Į +éľ ¾ +å¥ĩ 纳 +å¯ Ŀ +æ¤ İ +æŁ ¬ +æĸ¯ åŁº +åħ¬ ç«ĭ +è¨ ĵ +é£ Ļ +é© ¿ +åĤ µ +èĽ Ļ +ç¯ĩ ç«ł +åĪĨ æĶ¯ +ä¸Ĭ å¹´ +çŃ Ŀ +ç¼ ¤ +èĢģ æŧ +åĻ ¬ +æľ ¦ +èĥ § +æ¶Ī è²» +æĵ Ķ +æ¦ ´ +æ¿ Ĵ +ç³ ¯ +æ³ ¸ +æį Ĩ +ç» ļ +èµ İ +çIJ IJ +èµ Ĥ +æħ ® +æ² Į +çĦ Ļ +æĴŃ æĬ¥ +æ· ĩ +åĪĩ åħ¥ +çij ķ +çĸ µ +éģ ´ +ç¨ ļ +ç© © +èŀ ĥ +æ£ ķ +æĨ § +æĨ ¬ +ä¼ º +æ¯ Ĺ +æį į +æĬ ī +ç´ Ĭ +å¼ Ľ +æĭ Ń +æĹı èĩªæ²» +åĿ · +ç« ¶ +è© ³ +è¿Ħ ä»Ĭ +è° ´ +çŀŃ è§£ +æŁ ¿ +é¢ Ĭ +ç° § +çĥŁ èĬ± +ä¾ ¥ +çĿ ¦ +éħ Ŀ +æ° ĵ +çIJ ī +å§ Ĭ +æ² ® +æħ · +èľ ķ +çij ļ +éĩĩ çŁ¿ +åł ° +åºķ èķ´ +èĨ ³ +è¾ ķ +éŁ Ń +åĴ Ļ +ç² ½ +åī Ķ +æ² ¦ +èĤ ´ +éķ ¶ +æĺ ¼ +è¾ Ĺ +å© ª +åĮ ® +æĸ ĵ +æ± ¶ +éĥ ´ +éł » +çª Ĵ +è¢ ± +åĽ ± +èĢ ĺ +è ļĮ +çĭ Ļ +çĹ ¹ +ç¥ ī +æı ® +æ· Ĩ +ç£ ĭ +éĺ ª +æ « +ã ¸ +Ļ ¶ +ã ij +𣠲 +ä ¢ +ã Ń +𬠨 +ð¬ Ģ +𬠮 +𬠯 +ð¬ ľ +𪠨 +ð« Ĺ +ð¬ Ĭ +𬠱 +ð¬ Ł +ä İ +ð ¡ +ä ĥ +ã ł +ð © +ð© ¾ +𬠺 +ð¬ Ļ +ãĢ Ķ +ãĢ ķ +çļĦ æŶåĢĻ +æľīéĻIJ åħ¬åı¸ +ä¹ĭ åIJİ +ä¸ļ åĬ¡ +åķ Ĭ +èĻ½ çĦ¶ +æĭ¥ æľī +äºĴ èģĶç½ij +éĤ£ äºĽ +ä½ł çļĦ +åĨ³ å®ļ +éĻ¤ äºĨ +åĽ¢ éĺŁ +åı¯ æĺ¯ +以 åIJİ +社 åĮº +çļĦ éĹ®é¢ĺ +并 ä¸Ķ +æķĻ å¸Ī +å°± ä¼ļ +天空 éĥ¨èIJ½ +æľĢ ç»Ī +å½ĵ çĦ¶ +ä¹Ł æľī +ç¡® ä¿Ŀ +æĥ³ è¦ģ +è´Ń ä¹° +人 çļĦ +åIJ ´ +çļĦ åıijå±ķ +ä¸į çŁ¥éģĵ +软 件 +æĪij们 çļĦ +çĪ¶ æ¯į +åī ij +èĢĮ æĺ¯ +å®ī æİĴ +åIJİ æĿ¥ +çļĦ åľ°æĸ¹ +èµ µ +èĢĥ è¯ķ +çªģ çĦ¶ +ä¸Ģå®ļ è¦ģ +åĪ¶ ä½ľ +è¯Ħ ä»· +åħį è´¹ +è´¹ çĶ¨ +绣 ä¸Ģ +çĦ¶ èĢĮ +è¿Ļ 次 +éĿĴ å¹´ +人 ç±» +äº ¦ +让 人 +è´Łè´£ 人 +éĩĩ åıĸ +çļĦ äºĭæĥħ +ä¹Ł ä¼ļ +车 è¾Ĩ +æĽ´ æĺ¯ +强 åĮĸ +æĪij åĢij +以 åīį +ä¼ĺ åĮĸ +å§Ķåijĺ ä¼ļ +åĽ° éļ¾ +å¹´ 度 +ä½į äºİ +æĮĩ åĩº +åĨį 次 +åĬŀ çIJĨ +æ¯ı 个 +对 æĸ¹ +è¿Ľè¡Į äºĨ +æľĢ é«ĺ +课 ç¨ĭ +身 ä¸Ĭ +æĽ¾ ç»ı +åĮ» çĶŁ +å®ī è£ħ +æľ ± +è¿IJ è¡Į +åıĮ æĸ¹ +æľĢ 大çļĦ +æŀĦ 建 +è¿ŀ ç»Ń +çļĦ å°ı +她 çļĦ +çŃī çŃī +æĶ¹ åĸĦ +åIJĦ ç±» +éģĩ åĪ° +æľī çĿĢ +人 çī© +æĢ» æĺ¯ +è¿ħ éĢŁ +åĪ¶ å®ļ +å®ĥ 们 +å®ĺ ç½ij +è¿ĺ è¦ģ +ç»Ī äºİ +æĪ¿ åľ°äº§ +è¯ģ æĺİ +èĤ¡ 票 +åºĶ å½ĵ +èĭ± åĽ½ +è¿IJ çĶ¨ +æľĢ æĸ° +享 åıĹ +让 æĪij +æĻļ ä¸Ĭ +å¾ ŀ +å°ı 说 +å°¤åħ¶ æĺ¯ +è®Ń ç»ĥ +åħ¨ å¸Ĥ +æĮij æĪĺ +æľī çĤ¹ +带 çĿĢ +çļĦ ä¸ľè¥¿ +é£İ æł¼ +é»Ħ éĩij +å¼ķ 导 +æѤ å¤ĸ +æľĢ è¿ij +追 æ±Ĥ +强 è°ĥ +ä¹Ł åı¯ä»¥ +æĦŁ åĪ° +èĩª æĪij +çī¹åĪ« æĺ¯ +æĪIJ éĥ½ +éĢIJ æ¸IJ +å¿« ä¹IJ +ä¹ĭ ä¸Ń +æĬķèµĦ èĢħ +ä»ĸ们 çļĦ +æ° ı +å·¥ä½ľ 人åijĺ +äºĨ ä¸Ģ个 +åķ ¦ +ä¸Ģ åĢĭ +åŁº å±Ĥ +æ²Ł éĢļ +第ä¸Ģ 次 +并 没æľī +çļĦ å·¥ä½ľ +åľ¨ è¿ĻéĩĮ +æŀ ª +æĶ¯ æĴij +æŶ å°ļ +æĿ¥ åĪ° +æĶ¶ è´Ń +éĿ© åij½ +æĺ¯ ä¸įæĺ¯ +讨 论 +ä¸ļ 绩 +å°± èĥ½ +ç«ĭ åį³ +è¡Ĺ éģĵ +åľ¨ ä¸Ģèµ· +æľĪ 份 +é«ĺ 端 +å¾Ī éļ¾ +ä¿Ħ ç½Ĺæĸ¯ +æīĭ 段 +åģļ åĩº +ä¼Ĺ å¤ļ +å®ŀ è¡Į +æīĵ å¼Ģ +游 客 +ä¾Ŀ çĦ¶ +å°± åĥı +离 å¼Ģ +说 éģĵ +æĸ° èĥ½æºIJ +æº ª +äº ķ +令 人 +ä¸Ģ åľº +æĪij æĥ³ +两 人 +èĩ³ å°ij +çļĦ çĶŁæ´» +æĺ¯ ä¸ª +èĭ± è¯Ń +æ²Ĵ æľī +æĢĿ èĢĥ +éĻIJ åĪ¶ +åı° æ¹¾ +ä¸Ģ æŦ +çļĦ ä¸Ģ个 +é«ĺ 级 +åĬŀåħ¬ 室 +å¾· åĽ½ +æĪij å°± +å®ļ ä½į +éĢĤ åºĶ +æĮĩ æłĩ +åħ¨ çľģ +ä¸Ĭ è¿° +å®ĥ çļĦ +åĽŀ 家 +欧 æ´² +éĵģ è·¯ +é¼ĵ åĬ± +çļĦ å½±åĵį +é«ĺ æł¡ +天 ä¸ĭ +é«ĺ è´¨éĩı +æĿŃ å·ŀ +èµĦ 讯 +æĶ¾ åľ¨ +æľī ä¸Ģ个 +å°± è¦ģ +ä¸Ĭ éĿ¢ +解 éĩĬ +éĢIJ æŃ¥ +å°½ 管 +æľī ä»Ģä¹Ī +çļĦ äºĭ +çĻ» è®° +人æ°ij å¸ģ +è§Ĥ ä¼Ĺ +è§Ĥ å¯Ł +çĶµ èĦij +çļĦ åIJĮæŶ +ä½ľ ä¸ļ +宣 å¸ĥ +çļĦ ä½ľçĶ¨ +åĽŀ æĿ¥ +éļ¾ 以 +æīĢæľī çļĦ +å°ı åѦ +æıIJ åīį +æ¤į çī© +åĩ ¯ +ä¸Ĭ äºĨ +å°± åľ¨ +åħĪ åIJİ +æīĭ æľ¯ +éĥ Ń +éĿ¢ åīį +æ¯ķ ç«Ł +äºĮ æĺ¯ +红 èī² +éĺ³ åħī +èĭ¹ æŀľ +å¾Īå¤ļ 人 +ç»Ļ æĪij +åĵ ¦ +çľ¼ çĿĽ +éł Ń +ä¸Ģ æĺ¯ +åıijå±ķ çļĦ +åıį åºĶ +æĪ¿ å±ĭ +æľŁ å¾ħ +ç§į æ¤į +æĸĩ åѦ +åį³ åı¯ +é¦ĸ 次 +èĭ± éĽĦ +å¤ļ 次 +åĮħ è£ħ +æ²³ åįĹ +ä¹ĭéĹ´ çļĦ +ä»į çĦ¶ +åIJ¬ åĪ° +èij£äºĭ éķ¿ +è§Ħ åĪĻ +ä¸Ģ 份 +大 ä¼Ĺ +使 å¾Ĺ +è¿Ľ åı£ +ä¸Ģ çīĩ +æĢ§ çļĦ +çļĦ 大 +æĪij æĺ¯ +äºĴ åĬ¨ +æ° £ +çļ Ĩ +åħ¬åı¸ çļĦ +ä¸Ģ è¾¹ +åıĬ åħ¶ +èī¯ 好çļĦ +æĭĵ å±ķ +å½ĵ å¹´ +广 åľº +åģļ äºĨ +åŁº äºİ +æıIJ éĨĴ +åħĦ å¼Ł +èĢģ æĿ¿ +è¿ij æĹ¥ +çĬ¶ åĨµ +注 éĩį +åĪļ åĪļ +è°ĥ çłĶ +å¿ĥ ä¸Ń +æĬĬ æı¡ +éļı åIJİ +ä¸į å¤Ł +åĪĽ ä½ľ +ç«Ļ åľ¨ +缸 äºĴ +çĸ«æĥħ éĺ²æݧ +å¹´ 代 +带 åĬ¨ +伤 害 +ç«Ł çĦ¶ +å¼ķ è¿Ľ +ç´¯ 计 +让 æĪij们 +åĽŀ æĶ¶ +æĬ¥ åIJį +åĬ© åĬĽ +èģĶ 缣 +çŃĸ çķ¥ +åij¨ è¾¹ +åĭ Ĵ +è¿ĺ åľ¨ +æµģ éĩı +寻 æī¾ +çĶµ åĬĽ +èĪ¹ èĪ¶ +è¿ĺ èĥ½ +æĭħ ä»» +çļĦæĥħåĨµ ä¸ĭ +çļĦ åİŁåĽł +缺 ä¹ı +çIJĥ åijĺ +å²ģ çļĦ +çĶ· åŃIJ +å·¥ èµĦ +è¿ijå¹´ æĿ¥ +åij Ģ +æıIJä¾Ľ äºĨ +她 们 +家 åħ· +çĩ ķ +è½» æĿ¾ +æł¡ åĽŃ +èĢĥ æł¸ +åį± éĻ© +åħļ ç»Ħç»ĩ +æĢ» ç»ıçIJĨ +çļĦ æĸ° +çİ» çĴĥ +è¿Ļ ä½į +对 æѤ +家 人 +çļĦ è¦ģæ±Ĥ +温 度 +æĮĩ æķ° +缴 åĪ° +æѤ æŶ +æ¹ĸ åįĹ +éĥ½ è¦ģ +ä½ľ åĩº +åIJĦ ä½į +èĢĥ çĶŁ +ä¾Ŀ æį® +说 è¯Ŀ +æĪij ä¹Ł +å·¥ åİĤ +åıĺ æĪIJ +ä»ĸ 人 +æĪij è§īå¾Ĺ +åIJĦ 级 +ä¼łå¥ĩ ç§ģæľį +ä¸Ĭ åįĩ +好 åĥı +åĬł éĢŁ +äºĮ åįģ +è¢ ģ +è£ħ 饰 +éĥ½ èĥ½ +ä¸Ģ å¼ł +åĬ¨ æĢģ +å¹´ çļĦ +è¿Ļ å°±æĺ¯ +ä¹Ł è¦ģ +èµĦ æł¼ +æĪĺ äºī +æĦŁ è°¢ +åŁ¹ èĤ² +天 æ°Ķ +女 士 +åı¯èĥ½ ä¼ļ +çļĦ 产åĵģ +ä¹Ł å°± +主è¦ģ æĺ¯ +åĪº æ¿Ģ +ç»Ļ ä½ł +大 æķ°æį® +åĮ» åѦ +åĪ ¤æĸŃ +ä»ĸ 说 +表 æ¼Ķ +äºļ æ´² +ä¸ĵ é¢ĺ +ç«ŀäºī åĬĽ +éĤ£ æł· +å±ķ å¼Ģ +å¹³ æŶ +æİ¥ ä¸ĭæĿ¥ +æī¿ 诺 +æ³ķ åĽ½ +åħ³ å¿ĥ +ä¼ļ æľī +éĤĢ 请 +é¢Ħ éĺ² +对 æİ¥ +好 äºĨ +åĴ± 们 +çļĦ æĦŁè§ī +æĢĿ è·¯ +éĥ½ 没æľī +çļĦ æĸ¹æ³ķ +女 åŃIJ +åı¸ æ³ķ +è¿ĺ ä¼ļ +è¶ĬæĿ¥è¶Ĭ å¤ļ +åĽł çĤº +æµ· åįĹ +人 æķ° +å°Ĩ ä¼ļ +ä¸ļ 主 +é¤IJ 饮 +å±ħ ä½ı +åıij åĩº +è¿ij æľŁ +å¼ķ é¢Ĩ +æľºåĻ¨ 人 +åĩºæĿ¥ çļĦ +çľĭ è§ģ +ä¿ Ĭ +让 ä»ĸ +ä¸į æĥ³ +å·¥ä½ľ çļĦ +è¡¥ åħħ +æµ ħ +çī¹ å¾ģ +ä¸Ĭå¸Ĥ åħ¬åı¸ +ç¾İ é£Ł +广 西 +æ¯ı ä¸Ģ个 +èIJ½ åľ° +åĵģ ç§į +åĴĮ è°IJ +å½» åºķ +é«ĺ èĢĥ +æĺ¨ å¤© +åīį å¾Ģ +çĽij æµĭ +çĻ¾ 度 +åľ¨ ä¸ŃåĽ½ +çļĦ éľĢæ±Ĥ +亿 ç¾İåħĥ +åѦ æľ¯ +æĶ¶ åĪ° +æĿ¿ åĿĹ +ä¸Ģ 段 +æŀĦ æĪIJ +ä¼ģä¸ļ çļĦ +表 éĿ¢ +æķ´ çIJĨ +ç»ĵ å©ļ +人 家 +åģľ æŃ¢ +åѦ ç§ij +æĺ¾ å¾Ĺ +ä¼ij æģ¯ +é¢Ħ æľŁ +æĪĸ æĺ¯ +çļĦ 主è¦ģ +åºĶ 对 +èµ° äºĨ +ä¸Ń éĹ´ +èµ° è¿Ľ +åijĪ çİ° +æIJŃ éħį +é¹ ı +æĺ¯ åĽłä¸º +æĥħ 绪 +å®ļ æľŁ +社ä¼ļ 主ä¹ī +çŃī 级 +磼 çĽ¾ +é£ŀ æľº +èĩ³ ä»Ĭ +æĶ¶ éĽĨ +çļĦ æķħäºĭ +åĪĩ å®ŀ +å®ŀçİ° äºĨ +å½¢ æĪIJäºĨ +åįĹ æĸ¹ +ä¸Ń åѦ +æµ· æ´ĭ +åIJ¦ åĪĻ +æĭį æijĦ +大åѦ çĶŁ +åĩºçİ° äºĨ +æĦı å¤ĸ +ä¹Ł èĥ½ +çļĦ èĥ½åĬĽ +åĿIJ åľ¨ +åĪĻ æĺ¯ +èĢĥ å¯Ł +å°Ĭ éĩį +éĺ² æŃ¢ +ç´§ å¼ł +读 书 +åĩº è¡Į +å°± æľī +å±¥ è¡Į +çݰ代 åĮĸ +åĽ½ åĬ¡ +åĽ½åĬ¡ éĻ¢ +ç»´ ä¿® +åİŁ åĪĽ +æĺ¯ æĮĩ +ä¼ij éĹ² +çĤ ® +æĸ° æĹ¶ä»£ +éĢĻ åĢĭ +ä¸į æķ¢ +å®Į ç¾İ +ç»Ĩ èĬĤ +éŃ ı +èĶ¬ èıľ +é¢Ĩ导 çıŃåŃIJ +è¶ħ 级 +è¡Į æĥħ +人工 æĻºèĥ½ +åį° 度 +åŁºç¡Ģ 设æĸ½ +åıĪ æĺ¯ +èį¯ çī© +åIJ¸ æĶ¶ +åį´ æĺ¯ +éĥ İ +å¥ĸ åĬ± +çļĦ æľĭåıĭ +ä¿Ŀ çķĻ +è§Ħ å¾ĭ +æĸ° çĸĨ +è¿ĺ åı¯ä»¥ +æİ¥ è¿ij +æѤ åīį +æī¹ åĩĨ +æĢİä¹Ī æł· +çļĦ ä½įç½® +ä¸Ģ åĿĹ +æĭĴ ç»Ŀ +顾 客 +ä¹Ł åľ¨ +ä¸Ģ çĶŁ +éĥ¨ éĺŁ +å¹´ åīį +æĸ¹éĿ¢ çļĦ +å°Ŀ è¯ķ +羣æŃ£ çļĦ +ç¦ģ æŃ¢ +è¿ĺ 没æľī +æ°ij çĶŁ +èµ° åIJij +èĦ¸ ä¸Ĭ +å½ĵ 天 +éĽĨåĽ¢ åħ¬åı¸ +çļĦä¸Ģ ç§į +西 æĸ¹ +åĽŀ åºĶ +ä¸Ģ 声 +常 常 +æıIJ åĪ° +èħ¾ 讯 +æľį è£ħ +为 ä½ķ +äºij åįĹ +å°± ç®Ĺ +ä¼ł æī¿ +åıį èĢĮ +ä¸ĩ åIJ¨ +è´¢ 产 +å¦Ĥ ä¸ĭ +æĹ¥ åīį +åİŁ æľ¬ +æľĢ éĩįè¦ģçļĦ +认 è¯ģ +ä¸Ģ éģĵ +ä¿¡æģ¯ åĮĸ +å¾Ĺ åĪ°äºĨ +éĢ² è¡Į +æĪij è¦ģ +éĢļ ä¿¡ +室 åĨħ +èµļ éĴ± +æĶ¶ èĹı +解åĨ³ æĸ¹æ¡Ī +æĪ¿ 产 +çĭ ¼ +æ´» åĬĽ +ç»ıæµİ åıijå±ķ +çŃī å¾ħ +ä¹Ł å¾Ī +åĿ ij +å¾Ī 好çļĦ +éļ¾ 度 +ä¸į å¦Ĥ +人æ°ij æĶ¿åºľ +åĩº åıij +åīį æľŁ +æ¼Ķ åijĺ +女 çĶŁ +èģļ çĦ¦ +审 计 +é¢Ħ æµĭ +ä¾Ŀ æīĺ +äºĶ å¹´ +è¡¥ è´´ +æ¸ħ æĻ° +éª Ĥ +çľĭ èµ·æĿ¥ +çļĦ åŃ©åŃIJ +é¢ij éģĵ +ä½ı å®ħ +éĿ¢ åIJij +æľĢ ä½İ +æĹ¢ çĦ¶ +ä¸Ģ å¥Ĺ +æķ° åѦ +群 ä½ĵ +åĮĹ京 å¸Ĥ +å±ħ çĦ¶ +æ°Ľ åĽ´ +éĢĶ å¾Ħ +çļĦ åŁºç¡Ģä¸Ĭ +èģĮ è´£ +åı¯èĥ½ æĺ¯ +åĨĽ äºĭ +æĪIJ æķĪ +åŃ©åŃIJ 们 +计ç®Ĺ æľº +èµ ¤ +产ä¸ļ åıijå±ķ +å·¨ 大çļĦ +å·¥ 人 +çĶŁ éķ¿ +éĥ½ åı¯ä»¥ +çļĦ æľºä¼ļ +èµĦ è´¨ +çĹĽ èĭ¦ +ç²ī ä¸Ŀ +å¢ ĵ +å¹³ å®ī +管 éģĵ +è·Ł çĿĢ +饮 é£Ł +åķĨ 家 +å¤ļ 家 +åı¸ æľº +åºĶ该 æĺ¯ +éĢı éľ² +认 å®ļ +è¡Įä¸ļ çļĦ +çļĦ ä¼ģä¸ļ +æ¯ı ä¸Ģ +èĮĥåĽ´ åĨħ +è¾ĥ 大 +è´ ¤ +大 èµĽ +å¤ļ äºĨ +é¸ ¿ +临 åºĬ +åľ¨ è¿Ļ个 +çļĦ åĨħ容 +éĶĢ éĩı +å¾Ī å°ij +åŃ Ł +ç»´ æĮģ +åĴĸ åķ¡ +æľ¬ åľ° +èī² 彩 +并 éĿŀ +èĢĮ å·² +温 æļĸ +èIJ § +æĬĵ ä½ı +èĢĮ ä¸įæĺ¯ +åĸ Ĭ +çļĦ åħ³ç³» +çī© åĵģ +éĤ£ æĺ¯ +åĨľ 产åĵģ +è¿Ļ æŶ +å©ļ 姻 +æ°´ æŀľ +æĶ¶ èİ· +ä»ĺ åĩº +客æĪ· 端 +æ¼Ķ åĩº +åħ¨ æĸ° +è¿Ļ ä¹Łæĺ¯ +æĺ¯ çĶ± +è§Ĥ 念 +æľī 个 +éĢł åŀĭ +èĥľ åĪ© +ä¸ī æĺ¯ +è¶ħ å¸Ĥ +åħļ建 å·¥ä½ľ +æĶ¾ å¿ĥ +线 è·¯ +æĭĽ çĶŁ +åIJĥ é¥Ń +è½ ī +å°½ éĩı +è§ģ åĪ° +åIJĮæ¯Ķ å¢ŀéķ¿ +åįİ ä¸º +æĪij å¸Ĥ +æıIJ åĩºäºĨ +æ°ij èѦ +åįļ çī© +åįļçī© é¦Ĩ +è¯ļ ä¿¡ +åīį éĿ¢ +å±± 西 +è¾ħ åĬ© +转 移 +æĽ´ 为 +丰å¯Į çļĦ +åį ¢ +å¿« éĢĴ +æĺ¾ èijĹ +çī© èµĦ +åĪ° è¾¾ +æľī åĪ©äºİ +åij Ĩ +åŃ©åŃIJ çļĦ +ä¸į ä½Ĩ +çłĶ究 éĻ¢ +çĶ³ æĬ¥ +æļ ¨ +æ°ij éĹ´ +åį » +çļĦ å£°éŁ³ +å¸Ĥåľº çļĦ +ä¸Ģ åı¥ +çľģ 级 +æĿ¥ çļĦ +åĵª 个 +æīį ä¼ļ +åĪĨ éħį +èĶ ¡ +ä»ĸ åľ¨ +åħ± æľī +å¡ ĺ +èĴ Ĥ +éľ į +åıĤ è§Ĥ +ä¸Ī 夫 +ä¾Ŀ éĿł +æľī æŶ +äºĨ å¾Īå¤ļ +ä¸ĸçķĮ æĿ¯ +家 æĹı +ä¸į éľĢè¦ģ +大 å¸Ī +èŀį åħ¥ +éĿŀ æ³ķ +çĹħ 人 +åIJİ æľŁ +大家 éĥ½ +ç½ij åĿĢ +åİŁ æĸĻ +便 å®ľ +æ¶ Ľ +仿 ä½Ľ +å·® è·Ŀ +åı¦ä¸Ģ æĸ¹éĿ¢ +产åĵģ çļĦ +èµ « +æĥħåĨµ ä¸ĭ +éĴ¢ éĵģ +æľ¬ ç«Ļ +纳 åħ¥ +å·² æľī +æľī 没æľī +ä¼° 计 +é£ ĺ +æľŁ è´§ +åĢĭ人 è³ĩæĸĻ +ä¸ĵä¸ļ çļĦ +çĪĨ åıij +èĩ´åĬĽ äºİ +çİ°åľ¨ çļĦ +æľī åĵªäºĽ +çł´ åĿı +æķ°åŃĹ åĮĸ +åľ° éĿ¢ +é»ij èī² +å¹¼åĦ¿ åĽŃ +çļĦ ç²¾ç¥ŀ +äº Ń +导 æ¼Ķ +çİ° æľī +æѦ åĻ¨ +èĭı å·ŀ +çİ Ħ +æ±Ł 西 +延 伸 +论 æĸĩ +è¾ĥ 为 +çİ© æ³ķ +é¼ İ +åIJĮ æŃ¥ +éĩĬ æĶ¾ +æĽĿ åħī +åĿļ åĨ³ +å§Ķ æīĺ +å°Ĩ åľ¨ +äºĪ 以 +ä½ľ æĸĩ +èĢĮ åľ¨ +ä¼ĺ åħĪ +åĽŀ åİ» +ä¿® å¤į +åĽ½åĨħ å¤ĸ +çŃĸ åĪĴ +åıij æĶ¾ +å¿ĥ æĥħ +çļĦ åİĨåı² +éĿ¢ è¯ķ +举 åĮĹ +ä¿¡ åı· +ç²® é£Ł +è¯ģ 书 +æŁIJ äºĽ +è¿IJ ä½ľ +åĨ² åĩ» +çĥŃ çĤ¹ +æŶ æŶ +æŶæŶ 彩 +åľ° çĤ¹ +ä¸Ģä½ĵ åĮĸ +éļ¾ é¢ĺ +æĽ ° +ç«ĭ åĪ» +æĺ¯ éĿŀ常 +åħ± åĴĮ +åħ±åĴĮ åĽ½ +æ¿Ģ åĬ± +æľīæķĪ çļĦ +å¤Ħ ç½® +该 åħ¬åı¸ +æ£Ģ éªĮ +èѦ æĸ¹ +è´ ¾ +äºĨä¸Ģ ä¸ĭ +ä»Ĭ åIJİ +çħ ® +çĶ¨ åĵģ +读 èĢħ +æĪij åľ¨ +åĽŀ å¤į +ä¸Ģ 座 +è¿ĺ 没 +å®ļ åĪ¶ +没 æĥ³åĪ° +å¤ ¹ +ä¼ł éĢĴ +ä¸Ģ 款 +强 大çļĦ +çļĦ è¡Į为 +å¤ı 天 +åıijåĬ¨ æľº +é¢ĨåŁŁ çļĦ +å®ŀéªĮ 室 +ä¸Ģ æĬĬ +æĺ¯ ä¸ºäºĨ +éĻķ 西 +æĭħ ä¿Ŀ +è¾¾ æĪIJ +è¦ģ æĺ¯ +æĺİ å¤© +ç»Ļ ä»ĸ +建ç«ĭ äºĨ +ä¸į è¡Į +ä¸Ń æĸĩ +åľ° 说 +åIJİ çļĦ +çĽij æݧ +éĢ ¸ +æĢ» éĥ¨ +æľ¬ æĸĩ +é¹ ¿ +æĻ¯ è§Ĥ +çļĦ 缮æłĩ +èĽ ĩ +åĨ ¯ +ä¸Ń åĮ» +æķĪ åºĶ +产 éĩı +åŃ Ŀ +è´¦ æĪ· +è¿Ŀ åıį +èij£äºĭ ä¼ļ +京 举 +责任 ç¼ĸè¾ij +åķı é¡Į +çĪ± å¿ĥ +èѦ å¯Ł +é¤IJ åİħ +å¸Ĥ æĶ¿åºľ +天 天 +æĸ° é²ľ +éĥij å·ŀ +è¶ħ è¶Ĭ +å½ Ń +çŁ¥è¯Ĩ 产æĿĥ +åĽŀ å¿Ĩ +è·¯ 线 +å»ī æ´ģ +éĿĴ å°ijå¹´ +åıĸå¾Ĺ äºĨ +çľĭ åĪ°äºĨ +é¦ ¬ +ç²¾ åĵģ +åľ° éĵģ +æĮģ æľī +ä¸ĭ äºĨ +æľī æŶåĢĻ +ä¸Ģ 人 +æĴ Ĵ +ä»Ķ ç»Ĩ +èĢģ åħ¬ +äºĭå®ŀ ä¸Ĭ +èģĶ èµĽ +ä¾ĽåºĶ éĵ¾ +é¢Ħ ç®Ĺ +åĪ¶éĢł ä¸ļ +å®īåħ¨ çĶŁäº§ +俱 ä¹IJ +俱ä¹IJ éĥ¨ +çļĦ æł¸å¿ĥ +æīĵ ç®Ĺ +å½± çīĩ +æIJŃ å»º +ä¹Ł ä¸įä¼ļ +æĭħ å½ĵ +å±Ĥ éĿ¢ +åѦ åijĺ +临 æŶ +缸 ç»ĵåIJĪ +对 æ¯Ķ +ä»ĸ æĺ¯ +æĸ° åĮº +è¿Ľ åİ» +çĻ¾ å¹´ +ä¿ © +å°½ å¿« +çĶµåŃIJ åķĨåĬ¡ +æĽ´ æľī +æ¸ħ çIJĨ +åı¦ ä¸Ģ个 +åĤ » +ä»Ģä¹Ī æł·çļĦ +æĺ¯ æľĢ +åij¨ å¹´ +å¾Ī 容æĺĵ +åĽ¢ ç»ĵ +ç´ Ħ +æĹ© å·² +çļĦ åıĺåĮĸ +éľ ŀ +æĹ¥ ä¸ĬåįĪ +失 åİ» +ä¸Ń åľĭ +çļĦä¸Ģ äºĽ +å°ı åŃ© +ä¸ĭ è·Į +éĶ» çĤ¼ +é ij +éij « +å¿ĹæĦ¿ èĢħ +èĤ¡ å¸Ĥ +èµĽ äºĭ +许åı¯ è¯ģ +åı¯ æĮģç»Ń +åijĬè¯ī è®°èĢħ +éĢ» è¾ij +å¼ķ åħ¥ +çļĦ è¿ĩç¨ĭä¸Ń +è§Ĩ è§ī +èĩªæ²» åĮº +è¯ģ æį® +è£ħ ç½® +第ä¸ī æĸ¹ +å¹´ æĿ¥ +å¹¿ä¸ľ çľģ +带æĿ¥ äºĨ +éķ¿ æ±Ł +访 éĹ® +å·® ä¸įå¤ļ +æĺ¯ æĪij +éģŃ éģĩ +æĬĵ 好 +é«ĺ è¾¾ +并 åľ¨ +èĩª è§ī +ä¾ĽåºĶ åķĨ +æĥħ æĦŁ +ä½ı äºĨ +çļĦ èģĮä¸ļ +çļĩ å¸Ŀ +西 éĥ¨ +åĴĮ å¹³ +çļĦ åĬĽéĩı +æ± ª +åħħåĪĨ åıijæĮ¥ +æĬķ è¯ī +èµ· åĪ° +äºĴ 缸 +æ¾³ éŨ +æİ¥ åĪ° +æ°´ æ³¥ +模 åŀĭ +ä¸Ģ åįĬ +秩 åºı +æĪij们 åľ¨ +æī¿ 认 +ä¸Ģ éĥ¨åĪĨ +åįł æ¯Ķ +å¦ĩ 女 +ç² ĺ +äºĨ解 åĪ° +ä¸Ģå®ļ ä¼ļ +åIJĦ 大 +èµ° åĩº +为 大家 +é«ĺ éĵģ +åı¯ä»¥ åľ¨ +ä½Ĩ åľ¨ +çĶŁæĢģ çݯå¢ĥ +èı ¯ +çļĦ ä»·æł¼ +麻 çĥ¦ +æ¿Ģ åıij +éĤ£ å°± +çļĦ æł·åŃIJ +为 æѤ +天 åľ° +çļĦ 缮çļĦ +åĢº åĪ¸ +å·² ç¶ĵ +åĽĽ 大 +åIJĮæŶ ä¹Ł +å½¼ æѤ +æĭ¿ åĪ° +åIJ« éĩı +åįģ 大 +éļ¾ éģĵ +å¼ Ĺ +ä¸Ģ 段æŶéĹ´ +çħ§ 顾 +æķ°æį® æĺ¾ç¤º +æĪIJ为 äºĨ +èµ° åĪ° +æľ¬ åħ¬åı¸ +ç»Ī 端 +ä¹Ł ä¸įæĺ¯ +头 åıij +大 约 +é£İ æĻ¯ +æ¶Ī èĢĹ +审 æŁ¥ +äºī åıĸ +æ³ķ æ²» +äºĭ çī© +ç¼ĵ 解 +æĥ ¨ +缸åºĶ çļĦ +çļĦ æķĪæŀľ +åıį å¤į +åıijçĶŁ äºĨ +éĢĻ äºĽ +ç»ĥ ä¹ł +åݨ æĪ¿ +å¼Ģ æĭĵ +欣 èµı +夫 妻 +ä¸į ä¸Ģæł· +产 èĥ½ +èĬ¯ çīĩ +è¦ģ ç´ł +åıį 对 +çİĩ åħĪ +è´§ çī© +æĹ¥ çĶµ +ä½ľ 家 +æĶ¹ è¿Ľ +æĪIJ åĪĨ +åĽł èĢĮ +åĩı èĤ¥ +æ½ ĺ +å±±ä¸ľ çľģ +åĬ Ŀ +åŁ ĭ +æѦ è£ħ +æ±ĩ æĬ¥ +ä¸Ģ个 æľĪ +çĥŃ éŨ +大 éģĵ +æ´» åĭķ +éĥ½ å¾Ī +çĶµ 梯 +ç´§ æĢ¥ +åĢº åĬ¡ +客 æľį +ä¸Ģ éĥ¨ +ä½ł æĺ¯ +çİ° çĬ¶ +æŃ£ç¡® çļĦ +ä¹ĭ å¤Ħ +ç¼ĸ åĪ¶ +ä½ł åı¯ä»¥ +çŃī åľ° +èİ ī +对 è¯Ŀ +æ·ĺ å®Ŀ +è°ĥ èĬĤ +æİĴ æĶ¾ +åºĵ åŃĺ +ç´ ļ +çļĦ ä¼ĺåĬ¿ +æĿĥ å¨ģ +以ä¸ĭ ç®Ģ称 +ä¸Ģ 项 +èģļ éĽĨ +ä¼łç»Ł çļĦ +æ·· åIJĪ +è¿Ļä¸Ģ çĤ¹ +ä¸Ģ çľ¼ +æĹł éĻIJ +èİ·å¾Ĺ äºĨ +éĢī æīĭ +åĪ¶ åĵģ +åįı ä½ľ +çĭ¬çī¹ çļĦ +ä¸Ģ 级 +è¿Ļ个 éĹ®é¢ĺ +æĸ Į +æĺ¯ æĪij们 +æķĮ 人 +æ¸ħ æ´Ĺ +ä¸Ģ缴 åľ¨ +å°ı ç±³ +çļĦ è¿ĩç¨ĭ +åľ¨ åĮĹ京 +ä¸Ģ æĶ¯ +æĹ© ä¸Ĭ +æĸĩ èīº +ç¦ı åĪ© +é£Ł çĶ¨ +æĦŁ åĬ¨ +åħ¨ ç¨ĭ +æĶ¯ åĩº +æĸ° 建 +å¸ ķ +æĺ¾ çĦ¶ +羣 çļĦæĺ¯ +æĸ°éĹ» ç½ij +èĥ½ åIJ¦ +åįı åĬ© +亲 èĩª +å¾Ī æľī +çĻ¼ å±ķ +æĦı 大 +æĦı大 åĪ© +çĶµ ç½ij +æĹ¥ çĽĬ +çĨ ± +èĤĮ èĤ¤ +çĶ· æĢ§ +ç»Ħ 建 +çŃī éĹ®é¢ĺ +æ¶Ī éĻ¤ +æĬ¤ çIJĨ +å¡ij æĸĻ +ä¹Į åħĭ +ä¹Įåħĭ åħ° +åķĨ æłĩ +çIJ ³ +æĸ° æīĭ +çļĦ çī¹çĤ¹ +åĴ ¬ +å½ĵ ä¸ĭ +设计 å¸Ī +èµĶ åģ¿ +第 åįģ +æĻºèĥ½ åĮĸ +å¼Ģåıij åĮº +åı¯ä»¥ éĢļè¿ĩ +åħ±äº§ åħļ +åİī 害 +çģµ æ´» +æŶ åħī +éĥ¨ ä½į +人 æĸĩ +è¿Ľ æĿ¥ +ä¹ĭ æīĢ以 +ä¸ī åįģ +çļĦ åѦçĶŁ +éĺ² æĬ¤ +åĽ½ 产 +æ·±åľ³ å¸Ĥ +éĤ£ å°±æĺ¯ +åĪ° ä½į +çī¹ æľĹ +çī¹æľĹ æĻ® +å®ŀ æŶ +åı° çģ£ +èĢĮ ä¸į +æĮĩ å®ļ +åĿ Ŀ +èħIJ è´¥ +çī¹ å®ļ +å¢ŀ éĢŁ +æłĩ çŃ¾ +æĪ¿ ä»· +æĦ ģ +贯彻 èIJ½å®ŀ +æĢ§ è´¨ +çłĶ究 çĶŁ +ç¾İ 容 +æī¹ è¯Ħ +究 ç«Ł +人åĬĽ èµĦæºIJ +éĸĭ å§ĭ +åĽŀ å½Ĵ +èIJ¥ åķĨ +èIJ¥åķĨ çݯå¢ĥ +ä¸ŃåĽ½ 人 +çļĦ åŁºæľ¬ +è¯Ŀ é¢ĺ +æłĩåĩĨ åĮĸ +西 èĹı +åĭ ¾ +çļĦ 设计 +ç®Ģåįķ çļĦ +å¤į åĪ¶ +æ¸IJ æ¸IJ +以 å¤ĸ +èģĶ åĬ¨ +两 次 +æĢ§ åĴĮ +æĽ´ 大 +çļĦ åIJįåŃĹ +éŁ ¦ +ä½ł è¦ģ +å¢ĥ å¤ĸ +æĹ© æľŁ +åĪĿ æŃ¥ +è´¦ åı· +害 æĢķ +æĺ¨ æĹ¥ +åĪļ æīį +ç¥ŀ ç§ĺ +ç²¾ å¿ĥ +æµģ éĢļ +åħ¨ æĸ¹ä½į +以 å¾Ģ +ä¹Ł å°Ĩ +æĺ¯ ä¸ŃåĽ½ +åĽ½å®¶ 级 +å°Ĩ åĨĽ +æij Ĭ +æľĢ 为 +第ä¸Ģ æŶéĹ´ +æ¶Ī æ¯Ĵ +å°Ĩ äºİ +å¨ģ èĥģ +èĭ± æĸĩ +æīĭ ä¸Ń +çIJĥ è¿· +è§Ĥ çľĭ +离 å©ļ +æľ¬ åľŁ +åĪĨ æķ£ +æĻ ´ +è¦ģ 注æĦı +浪 è´¹ +管 æݧ +åĩº åĶ® +æĢ» è£ģ +ä¸Ģ éĺµ +å¨ ĩ +äºĶ 个 +å½ĵ åĪĿ +çºł 纷 +ä¸ĵ çĶ¨ +å¤ĩ æ¡Ī +åĪĿ æľŁ +å®ĥ æĺ¯ +åĮº åĿĹ +åĮºåĿĹ éĵ¾ +大 è¿ŀ +è¿Ļ ç±» +åıĺ æĪIJäºĨ +éĤĦ æĺ¯ +åįļ 客 +çı¾ åľ¨ +ä¸Ģ æĸ¹ +å®ĮæĪIJ äºĨ +è¿Ļ个 æŶåĢĻ +åħ¨ å¹´ +ä¸Ĭ 线 +ç½ IJ +ç«ŀ èµĽ +åĩºçīĪ 社 +åĵ¥ åĵ¥ +å¯ « +å¾Ĺ 以 +èĬ± åĽŃ +äºĨ èµ·æĿ¥ +èĦ±è´« æĶ»åĿļ +çļĦ åİŁåĪĻ +讲 解 +æ¶Ī åĮĸ +æįŁ 害 +æļĤ æŶ +å¾Ĺ çŁ¥ +éĢĤ çĶ¨ +éŨ åºĹ +解 读 +æĻ® åıĬ +人æ°ij æ³ķéĻ¢ +åī¯ 主任 +å¿ĥ çģµ +è¯Ĭ æĸŃ +ç¾İ 女 +æŁ ¯ +å¹´ 以æĿ¥ +æ´» è·ĥ +åĢŁ åĬ© +åħ± 建 +è¯ī 讼 +æĶ¾ æĿ¾ +çªĹ åı£ +ä¼ģ æ¥Ń +åĬł æĭ¿ +åĬłæĭ¿ 大 +ä¹° äºĨ +主 æµģ +æĩĤ å¾Ĺ +å°Ĩ åħ¶ +éĢı æĺİ +å·¥ä½ľ ä¸Ń +èĤ¡ ä»· +æ¡£ æ¡Ī +没æľī ä»»ä½ķ +åijĬ çŁ¥ +å¹´ åĪĿ +æĹ¥ ä¸ĭåįĪ +åİĤ åķĨ +èĬĤ å¥ı +主 导 +è£ Ŀ +åħ³éĶ® è¯į +èģĬ 天 +åĨĻ ä½ľ +æĶ¹éĿ© å¼ĢæĶ¾ +æľī æľĽ +éĢļ æĬ¥ +èIJ Į +æĢ» é¢Ŀ +çŁŃ æľŁ +ä¸Ģ çķª +çĶŁæ´» çļĦ +åĮĸ çļĦ +æĺ¥ å¤© +è¿Ļ åľº +æĸ°å¼Ģ ä¼łå¥ĩ +æĺ¯ è¦ģ +å°ļ æľª +åıĺ æĽ´ +ä¸Ģ åij¨ +客 è§Ĥ +æĹ¥ èĩ³ +é¹ ° +çİ ² +å°Ĩ æĿ¥ +客 人 +åıĺ éĿ© +说 äºĨ +åİŁ çIJĨ +èģĮ åĬ¡ +åıĪ æľī +ä¸Ģ åı¥è¯Ŀ +æĦŁ åıĹåĪ° +ç¬Ķ èĢħ +移 æ°ij +西 åįĹ +ä¹ĥ èĩ³ +æŃ£ è§Ħ +åĪĿ ä¸Ń +çĬ ¬ +å½ĵ äºĭ +å½ĵäºĭ 人 +æĪij们 è¦ģ +åħ¥ åı£ +éĤ£ æŶ +æľīéĻIJ 责任 +å°ij 女 +è¿Ļä¹Ī å¤ļ +åĪĨ åħ¬åı¸ +å®ĩ å®Ļ +çļĦ éĢīæĭ© +å§IJ å§IJ +åıij èµ· +è» į +æĽ´å¥½ åľ° +éĻĨ ç»Ń +æľ¬ æľįåĭĻ +å« © +赶 ç´§ +èĦĤ èĤª +第äºĮ 天 +æĪij ä¼ļ +两 ä½į +æķ ² +åħ¬å®ī æľºåħ³ +ç§ijæĬĢ åĪĽæĸ° +å°º 寸 +è¾IJ å°Ħ +å®Ĺ æķĻ +转 æį¢ +åĩº çİ°åľ¨ +ä¸Ģ é¢Ĺ +æľŁ éĻIJ +åIJĮåѦ 们 +åĮĹ æĸ¹ +ä½ł å°± +ä¸Ģ带 ä¸Ģè·¯ +èĢģ å©Ĩ +游æĪı çݩ家 +çļĦ ç»ĵæŀľ +è¡¥ åģ¿ +å¤ĸ è´¸ +对 å¾ħ +ç»´ çĶŁç´ł +ç»ıéĶĢ åķĨ +è¿ĺ å°Ĩ +åŃIJ 女 +æĽ´ é«ĺ +ä¸į 大 +éī´ å®ļ +让 ä»ĸ们 +æīĢè°ĵ çļĦ +æŃ» äºĨ +帮 æī¶ +åĵ² åѦ +以ä¸Ĭ çļĦ +çļĦ åħ³éĶ® +æĹ© å°± +æĬ¥ ä»· +éģµ å®Ī +æī© å¼ł +æĺ¯ å¾Ī +å¼Ģ éĢļ +æĸ° åĬł +æĸ°åĬł åĿ¡ +ç¿» è¯ij +询 éĹ® +é¸ Ń +ä½ĵ åĨħ +两 个人 +çĪ ¹ +éľ ľ +乡æĿij æĮ¯åħ´ +çĿ¡ è§ī +å®ĺ åijĺ +åĪĽ å§ĭ +åĪĽå§ĭ 人 +ä¼Ĺ 人 +åį³ 便 +çĸ« èĭĹ +ä¼ģä¸ļ 家 +æ¸ £ +ç²¾ åĬĽ +å¤ĸ éĥ¨ +èģª æĺİ +è¿Ļ ä¹Ł +å½ķ åıĸ +åĨ² çªģ +åħ¨ 身 +åŃ£ èĬĤ +忽 çĦ¶ +çļĦ æĢģ度 +åĤ¨ å¤ĩ +ä¿Ŀ åħ» +çļĦ æĥ³æ³ķ +ä¸Ĭæµ· å¸Ĥ +æIJº æīĭ +çļĦ ä¿¡æģ¯ +åķĨ åľº +çļĦ æĢĿæĥ³ +æĿĥ åĬĽ +毫 æĹł +æĢĢ åŃķ +硬 件 +åĨħ èĴĻåı¤ +æİ¢ 讨 +åħ» çĶŁ +çļĦ 表çİ° +空 ä¸Ń +æģIJ æĢĸ +å¾Ī é«ĺ +ç»ıæµİ 社ä¼ļ +ä¸Ĭ æĿ¥ +延 ç»Ń +éĩį å¤į +éĺ² èĮĥ +çļĦ å½¢å¼ı +æľĪ åºķ +èĢģ 年人 +绿 åĮĸ +å±± åĮº +æĭ¿ åĩº +æĹħ 客 +æĽ´ æį¢ +åħ¬ 主 +èĬĤ 约 +åħ¨ åİ¿ +åĽŀ æĬ¥ +çIJĨ æĢ§ +çĸ¯ çĭĤ +æ¶ī å«Į +åī§ æĥħ +åĨ¬ åŃ£ +åIJİ ç»Ń +è¿Ļæĺ¯ ä¸Ģ个 +æ¼Ķ 讲 +ä¸Ģ å±Ĥ +æľīåħ³ éĥ¨éŨ +æĹł å¥Ī +ç§į ç±» +缸åħ³ çļĦ +æĪĸèĢħ æĺ¯ +æī¶ æĮģ +å¤ļ æķ° +çļĦ ä½ľåĵģ +ä¸ĭ ä¸ĢæŃ¥ +å¸Ī åĤħ +é«ĺéĢŁ åħ¬è·¯ +好 åıĭ +ä¼ĺç§Ģ çļĦ +è¿Ľ äºĨ +æģIJ æĢķ +äºĨ åIJ§ +大 è§Ħ模 +çļĦ ä¸ĸçķĮ +æĢĢ çĸij +å· · +åħ´ å¥ĭ +æĪ ° +æĿij éĩĮ +æľĭåıĭ åľĪ +åĨ¬ 天 +ä¸Ńåįİ äººæ°ij +åįı åķĨ +è¯Ħ éĢī +æĹ Ń +å¢ŀåĬł äºĨ +åıĹ ä¼¤ +ä¸Ģ èĤ¡ +便 æį· +ä¸ ij +é¹ ¤ +å¤ĸ è§Ĥ +å·¥ç¨ĭ å¸Ī +åĴĮ åħ¶ä»ĸ +è¿Ļ å°± +ä¸Ńå°ı ä¼ģä¸ļ +西 åĮĹ +åĽ½æľī ä¼ģä¸ļ +èĭ¥ æĺ¯ +åı¯ æĥľ +çĶŁ æĹ¥ +åĩ ½ +ä¹° åįĸ +ç¥Ŀ ç¦ı +人æ°ij 群ä¼Ĺ +åħī æĺİ +åħ¬ å¯ĵ +æĺ¯ è°ģ +æĪij çŁ¥éģĵ +è¯Ń æĸĩ +æķı æĦŁ +ä¸įéĶĻ çļĦ +æĿ¥ 讲 +æ³¢ åĬ¨ +çļĦ 第ä¸Ģ +åľ° éľĩ +åľ¨ åħ¨åĽ½ +骨 å¹² +å®ī ç½® +家 çĶµ +ä¸İ æѤ +ä¸İæѤ åIJĮæŶ +åıĹ çģ¾ +çĥŃ çº¿ +çļĦ æĬĢæľ¯ +æµĭ éĩı +ä¾Ŀ èµĸ +ä¸ŃåĽ½ çļĦ +çī¹ æĢ§ +è¾ĥ é«ĺ +è¸ © +ä¼ļ åľ¨ +建 éĢł +导 èĪª +æĥ³ èµ· +åħ¨ ä¸ĸçķĮ +建 æĿIJ +ç¯ Ģ +çļĦ åŁºç¡Ģ +èĩªåĬ¨ åĮĸ +åīį åIJİ +çĿ¡ çľł +æݨ è¡Į +æį® äºĨ解 +ä»Ģä¹Ī æŶåĢĻ +ä¸į åĸľæ¬¢ +çħ¤ çĤŃ +éĤ£ä¹Ī å¤ļ +å¸Ĥåľº åĮĸ +ä¸į管 æĺ¯ +ç«ĭ åľº +éĥ½ 没 +课 é¢ĺ +æĪij们 å°Ĩ +è¿ĩ çļĦ +åĨį åĬłä¸Ĭ +çĪ ¾ +身 æĿIJ +çĶ· 女 +è¿ľ è¿ľ +çĶ· çĶŁ +èĩªèº« çļĦ +è´Ł æĭħ +çĻ¾ ä¸ĩ +西 çıŃ +西çıŃ çīĻ +åĩĢ åĪ©æ¶¦ +æ¾³ 大 +澳大 åĪ©äºļ +ä¸į åİ» +æī¿ åıĹ +楼 çĽĺ +å¢ĥ åĨħ +æ·· åĩĿ +æ··åĩĿ åľŁ +æĢĿæĥ³ æĶ¿æ²» +å¸Ĥ åĮº +æĭĽ æłĩ +åĽ¢ ä½ĵ +è¿Ľ 度 +åĨĽ éĺŁ +åıį å¼¹ +äºĨä¸Ģ äºĽ +æİ¥ å¾ħ +çļĦ åŃ¦ä¹ł +éħį éĢģ +é£Łåĵģ å®īåħ¨ +æĽ¿ 代 +æĺ¯ ä»¥ +éĢļ çĶ¨ +çłĶ究 æīĢ +ç¦ ħ +æī Ķ +éļĶ 离 +ä¸ĩ å¹³æĸ¹ç±³ +çļĦ è§Ħå®ļ +ç»Ļ æĪij们 +æ¿Ģ åħī +ä¼ļ åĩºçİ° +çŁŃ ä¿¡ +ç©¿ çĿĢ +æ²Ī éĺ³ +æķĻ æĿIJ +éĺ² çĸ« +ä¼ĺ èī¯ +约 å®ļ +æĪij çľģ +åħ¬ æ°ij +éģ¸ æĵ +éģ¸æĵ ĩ +å·² æĪIJ为 +ä¸į å¿ħ +ç¥ĸ åĽ½ +并 æľª +åľŁ 壤 +å¾® ç¬ij +äºĭä¸ļ åįķä½į +çļĦ 游æĪı +åħ¬ 示 +åIJĪçIJĨ çļĦ +çª Ŀ +æ°Ķ 象 +家 ä¸Ń +亮 缸 +åį« æĺŁ +è®° è½½ +è§Ĩ éĩİ +åľ°åĮº çļĦ +ä½Ĩ ä»ĸ +èĤĮ èĤī +äºı æįŁ +åĬŀ åѦ +ä¸Ģ è¡Į +è¯ŀ çĶŁ +åıijå¸ĥ çļĦ +çļĦ æľįåĬ¡ +çļĦ çłĶ究 +åij¨ æľ« +产ä¸ļ åĽŃ +é«ĺ 温 +æĪIJåĬŁ çļĦ +æŃ¥ 骤 +åŃĺ åĤ¨ +åŃIJ åħ¬åı¸ +让 她 +ä¸Ń æľī +åĺī 宾 +å¦ ® +æĺİ å¹´ +äºĨ åIJĹ +äºī è®® +æĪ Ī +ä¸Ģ æľ¬ +ç¾İ丽 çļĦ +ä½ł 说 +大 人 +æĶ» çķ¥ +ä¸į æľĥ +å¾ħ éģĩ +ä¸Ģ è¾Ĩ +çīĪæĿĥ æīĢæľī +æ°ij ä¼Ĺ +åĬŁ 夫 +å±ķ ä¼ļ +大 èĦij +æ¯ı æľĪ +å°ı 麦 +æµĻæ±Ł çľģ +çļĦ æīĢæľī +ä¸ĭ æ»ij +èĵĿ èī² +è¦ģ æĥ³ +åѦçĶŁ çļĦ +å½ĵ ä½ł +ä½ľ æĪĺ +家 乡 +å¤ļ åIJį +é«ĺ äºİ +åĿļ 强 +è¿ŀ éĶģ +åIJİ æŀľ +人 äºĭ +ç´ ħ +æ¿Ģ åĬ¨ +è¿Ľ æĶ» +ç© Ĩ +ä¸ ĺ +让 èĩªå·± +以 æѤ +夫 人 +å¼Ģ 设 +æ°Ķ è´¨ +鸡 èĽĭ +çĦ¡ æ³ķ +åIJĥ äºĨ +åĪĨåĪ« 为 +èģĶåIJĪ åĽ½ +å½ĵ 代 +å¦Ĥæŀľ æĺ¯ +è¿ľ ç¨ĭ +åĸ Ĥ +è®° ä½ı +æ¸ħ åįķ +åIJĪä½ľ ä¼Ļä¼´ +åİ» åģļ +æķħ éļľ +模 æĭŁ +å¸Ī çĶŁ +åīį æĿ¥ +çĶµè§Ĩ åī§ +çĥŃ çĪ± +éľ² åĩº +é«ĺ å±Ĥ +çĶµ åĻ¨ +纪 å¾ĭ +å¼Ģåıij åķĨ +éķ¿ å®ī +è½½ ä½ĵ +çļĦ å°±æĺ¯ +被 人 +åıĹ çIJĨ +篮 çIJĥ +èİ İ +交 ç»Ļ +æľªæĿ¥ çļĦ +两 大 +åIJķ å¸ĥ +çŃī 人 +çļĦ æĹ¥åŃIJ +åIJĪä½ľ 社 +æĮij éĢī +åŃĺ 款 +ç³»ç»Ł çļĦ +æĬĬ å®ĥ +没æľī ä»Ģä¹Ī +ä»İ æѤ +ä¸Ń åįĪ +çĸ¼ çĹĽ +å·© åĽº +浪 漫 +缸åħ³ éĥ¨éŨ +éķ¿ åŁİ +纤 ç»´ +ä¸Ĭ éŨ +çĪĨ çĤ¸ +èµ· çĤ¹ +çļĦ éĢļçŁ¥ +èĢĮ æĿ¥ +çļĦ èĢģ +æīĭ éĩĮ +è¯Ń éŁ³ +è¾Ľ èĭ¦ +æ±Łèĭı çľģ +çĶ¨ äºĨ +身份 è¯ģ +æľī åĬ© +æľīåĬ© äºİ +çī© èģĶç½ij +åĩº éŨ +å¼Ł åŃIJ +æĥ ¹ +è¿Ļ件 äºĭ +æĪij们 åı¯ä»¥ +çļĦ çĶŁåij½ +æľīä¸Ģ ç§į +åºĹ éĵº +åıĮ æīĭ +çļĦ æ¶Īæģ¯ +èĢIJ å¿ĥ +å°´ å°¬ +éĤ£ 天 +é¦ĸ æī¹ +æĺ¯ä¸Ģ 家 +人 æ°Ķ +åıį æŃ£ +æĪij åĴĮ +å®ł çī© +ä¸į 对 +寻 æ±Ĥ +缸 ä¼¼ +åľ¨ ç¾İåĽ½ +åı« åģļ +åĹ İ +ç«ĭ 足 +çĶ¨ éĢĶ +åħ Ĩ +大 æ°Ķ +åIJij ä¸Ĭ +ä»ĸ å°± +é¡¹çĽ® 建设 +èĭ¥ å¹² +æĺ¯ æľī +æ¿Ģ æĥħ +çļĦ æĦıä¹ī +æĺ Ń +严éĩį çļĦ +å¯Ĩ éĽĨ +èĪŀ è¹Ī +èį£ èİ· +èİ· æĤī +æ±Ł åįĹ +åģĩ å¦Ĥ +æĪ· å¤ĸ +线 ç´¢ +ç§ģ 人 +转åŀĭ åįĩ级 +çļĦ ä»·åĢ¼ +åįķ çĭ¬ +èĢģ çĻ¾å§ĵ +å°į æĸ¼ +åĽ½éĻħ åĮĸ +ä¼° åĢ¼ +æľįåĬ¡ ä¸ļ +èĩ Ń +æİī äºĨ +解åĨ³ äºĨ +ä¹Ł ä¸įèĥ½ +åħ ¹ +æĸ¯ çī¹ +æķħ æĦı +è¿ĩ 度 +èĬĤ æĹ¥ +çĻ½ çĻľ +çĻ½çĻľ é£İ +继 æī¿ +äºĨ ä¸įå°ij +äºĮ 人 +è§ģ éĿ¢ +æĥ³ æĥ³ +å¤į åIJĪ +康 å¤į +åİ¿ åŁİ +åľ¨ åĽ½åĨħ +åľº åľ° +éĻ¶ çĵ· +è¿Ļ 项 +çľ¼ ä¸Ń +çł ¸ +æĦŁè§ī åĪ° +æŀľ çĦ¶ +æĶ¾ åħ¥ +约 æĿŁ +æİĴ æŁ¥ +车 主 +çļĦ æĦıæĢĿ +æĸ° åŁİ +æĥ³ çĿĢ +éģ Ĥ +èĮ¶ åı¶ +ä¹° æĪ¿ +åĨľ æĪ· +é«ĺ æīĭ +çİī ç±³ +æĸ°åĨł èĤºçĤİ +çħ§ æĺİ +æĮĩ åįĹ +è¸ ¢ +æķij æı´ +æĻ¯ çĤ¹ +ç¨İ æĶ¶ +çļĦ æīĭ +æŃ£ 好 +è¦ģ æĬĬ +éļı æĦı +åħ¶å®ŀ æĺ¯ +ç»Ļ èĩªå·± +è°Ī åĪ¤ +æ¯ı天 éĥ½ +æĢģ åĬ¿ +é¢Ħ 约 +åİĨåı² ä¸Ĭ +å®Ŀ è´Ŀ +åīį è¿Ľ +ä¹Łå°±æĺ¯ è¯´ +çļĦ æĦıè§ģ +åı£ 罩 +åİĺ ç±³ +èĬ± è´¹ +ä½ĵèĤ² æĬķ注 +åħ¬ä¼Ĺ åı· +èijĹåIJį çļĦ +å¼Ģ æĪ· +æĭį åįĸ +å²ģ æľĪ +åĨħ 涵 +å®Įæķ´ çļĦ +é«ĺ åİĭ +åħ¬åĬ¡ åijĺ +使çĶ¨ çļĦ +çĶŁäº§ 线 +妹 妹 +èµ° 访 +æĺ¯ åı¯ä»¥ +åľ¨ 家 +æļ´ åĬĽ +æ³° åĽ½ +è´¨ çĸij +ä¸į éģİ +天çĦ¶ æ°Ķ +缺 çĤ¹ +å°ı åŀĭ +ä¸įä»ħ æĺ¯ +é»ij æļĹ +æ¢ ¨ +æĸĩ æĹħ +è¦ģ æľī +ä¸Ń å±± +çļĦ æķ°æį® +å¾Ĺ å¾Ī +以 便 +对 ä»ĸ +åĬł 以 +çĻ¼ çı¾ +设 å®ļ +èĤļ åŃIJ +éĿ ĸ +å¥ī çĮ® +ä¸į åıĺ +åı£ ç¢ij +åľ¨ åĵªéĩĮ +ä½ IJ +è¿Ļ 两个 +çļĦ æĸ¹åIJij +æŀ « +äºĮ 次 +çīĩ åĮº +éł IJ +ç£ Ĭ +æĭ¿ çĿĢ +å·²ç»ı æĪIJ为 +ä¹ĭ ä¸Ĭ +å®Ĺ æŨ +奶 奶 +é«ĺæĸ° åĮº +社 æľĥ +è·Ł 踪 +æľįåĬ¡ ä¸Ńå¿ĥ +æī ¯ +æīĭ æĮĩ +礼 çī© +宿 èĪį +çĶ¨ å¿ĥ +æıIJé«ĺ äºĨ +亮 çĤ¹ +ä¸į æĦ¿æĦı +æĴŃ æĶ¾ +å¤ļå°ij éĴ± +没 ä»Ģä¹Ī +æķ° åįģ +æĢ» çĽij +çļĦ åŁİå¸Ĥ +æī¾ åĪ°äºĨ +åĨħ åľ° +åĪ° çİ°åľ¨ +æĪĺæĸĹ åĬĽ +åİŁ å§ĭ +åĥ § +åĢĴ æĺ¯ +æľĢ åħ· +è´«åĽ° æĪ· +éĢģ åĪ° +级 åĪ« +åĩº èµĦ +æĪª æŃ¢ +ç§į åŃIJ +èĥ½ ä¸įèĥ½ +幸 è¿IJ +èĸ ĩ +项 éĵ¾ +æĮĤ çīĮ +ä¸Ģ 樣 +ä¹ĺ 客 +èIJ½ åIJİ +ä½Ĩ æĪij +æĹ© åľ¨ +åĬ¨ 漫 +å¹³ çŃī +对 ä½ł +ä¸į æĢķ +å¤ĸ çķĮ +å¤ļå¹´ æĿ¥ +é¦ĸ 个 +æ²³ åįĹçľģ +æĪĸ åħ¶ä»ĸ +éķľ 头 +åįĹ æĺĮ +ä¸Ģ éĿ¢ +éĢłæĪIJ çļĦ +å´ Ķ +çŃ Ĵ +æķĻèĤ² éĥ¨ +åľ° åŁŁ +æĺĨ æĺİ +å·´ é»İ +æīĭ 游 +ä¸Ģ æŶ +çł į +顶 级 +åħ± 计 +åİŁ æ²¹ +è¾ī çħĮ +说 æĺ¯ +æĸ°åįİ ç¤¾ +ç»ıåİĨ äºĨ +ä¸į æŃ¢ +è¦ģ ä¹Ī +èĢħ çļĦ +æĢ» æĬķèµĦ +è¡Į 驶 +ä¸Ĭ å¸Ŀ +å¹´ 纪 +çIJ ¼ +ä¼ł 说 +ç²¾ èĭ± +æĸ¹ éĴĪ +æ±Ł æ¹ĸ +æĪIJ çĤº +æĢ» éĩı +æĬķ æĶ¾ +åĬ¨ çĶ» +èĹ ¤ +çĶµ æºIJ +éĴ Ļ +åIJĮ è¡Į +æĻ®éĢļ çļĦ +åĽ¾ä¹¦ é¦Ĩ +è¯Ī éªĹ +æħĪ åĸĦ +è¿Ļ 份 +主æĮģ 人 +å°± è¿Ļæł· +èĢĮ æĪIJ +èĩªè¡Į 车 +ä¸ŃåĽ½ çī¹èī² +èĤ¿ çĺ¤ +åIJ ¾ +å¼Ł å¼Ł +åıĹ çĽĬ +éĢīæĭ© äºĨ +æĺİæĺ¾ çļĦ +æĬ¥ èĢĥ +ç¬ij éģĵ +éĽĸ çĦ¶ +温 å·ŀ +éĿŀ æ´² +ç§į ç§į +åıĤåĬł äºĨ +è´§ è¿IJ +éļı 便 +å°± 没æľī +ç¸ £ +央 è§Ĩ +ç©¿ è¶Ĭ +çļĦ çݰ象 +åĩł 次 +çļĦ é£İéĻ© +æŃĮ æĽ² +æľ¬ å±Ĭ +å¹´ åĨħ +ä¸į è¶ħè¿ĩ +è¿ĩ å¤ļ +å¿ħé¡» è¦ģ +ç»ĵ 论 +åĢŁ éī´ +ç¥ŀ å¥ĩ +æľŁ æľĽ +ä¸ĵ 享 +éĿŀ常 éĩįè¦ģ +æĦıè¯Ĩ åĪ° +åIJĪ 并 +æĬĬ èĩªå·± +å¥Ĺ è£ħ +éŃĶ æ³ķ +å¤ı åŃ£ +ä¸į åĥı +å¢ĥ çķĮ +æĥĬ åĸľ +æľīä¸Ģ 天 +çĦ¦ çĤ¹ +æĪij 认为 +åħ° å·ŀ +çĶµ æ°Ķ +èģĶç³» æĪij们 +ç§ij æĻ® +她 说 +çļĦ æĸĩç«ł +å¥ĩ æĢª +åıĭ 好 +饮 æĸĻ +çļĦ æĶ¯æĮģ +çŃĶ åºĶ +éĩį éĩı +çij ¶ +åĩı è½» +ç§ijåѦ 家 +å·´ 西 +éĩijèŀį æľºæŀĦ +åħļ å§Ķ书记 +貸 款 +ç²¾ èĩ´ +ä»İ æľª +åį° åĪ· +åĽŀ 顾 +é¦ĸ éĥ½ +åıij èĤ² +éĹ® éģĵ +è¾¾ åĪ°äºĨ +å¿į ä¸įä½ı +æīį æľī +æįIJ èµł +ä½Ľ æķĻ +ä¸į æ¸ħ +éĺŁ éķ¿ +缸 åıį +æĬ¥ èѦ +大 åħ¨ +欧 缣 +帮 å¿Ļ +çļĦ æĻĤåĢĻ +缮 å½ķ +足 以 +èī° éļ¾ +ä»ĸ ä¹Ł +å·¥ ä½ľèĢħ +头 èĦij +缺 éĻ· +æĪIJç«ĭ äºĨ +å°± å¼Ģå§ĭ +认 åIJĮ +é»Ħ èī² +çĹħ æĥħ +覺 å¾Ĺ +è¿Ļ 两 +ä¿¡ ä»° +åľĭ 家 +ä¸įä»ħä»ħ æĺ¯ +çĭ¬ 家 +èĪ¬ çļĦ +æĿIJ è´¨ +æµ· ä¸Ĭ +çĤº äºĨ +æľºåĬ¨ 车 +缸å½ĵ äºİ +å¤ļåħĥ åĮĸ +æĽ´ 大çļĦ +èĽ ® +åģĩ æľŁ +å¼ı çļĦ +交éĢļ è¿IJè¾ĵ +çľģ å§Ķ +ä¸į ç®Ĺ +æĶ¾ ä¸ĭ +éĹ ¯ +人 åľ¨ +港 åı£ +æŨ åľ¨ +åij½ 令 +æŁIJ 个 +å¹³ 稳 +åıª 好 +人 人 +äº ŀ +äºĮ ç»´ +äºĮç»´ çłģ +æŀģ 为 +åĪ« å¢ħ +åħ¶ ä½Ļ +大 äºĭ +主管 éĥ¨éŨ +æĹł éĶ¡ +éĹ µ +éģŃ åĪ° +说 è¿ĩ +为 ä½ł +解 çŃĶ +éªĮ æĶ¶ +çļĦ ç»ıéªĮ +åĮ¹ éħį +çģ« ç®Ń +豪 åįİ +æŁIJ æŁIJ +çļĦ æĹ¶ä»£ +书 éĿ¢ +æģĴ 大 +延 éķ¿ +ä¸Ģ åIJĮ +æľª èĥ½ +交 æį¢ +çĶ¢ åĵģ +çŃī åĪ° +åĪĨ 离 +æīĵ çĶµè¯Ŀ +å¹² çĩ¥ +è¾ĥ å¤ļ +å¤ļå¹´ çļĦ +èĥĮæĻ¯ ä¸ĭ +为 ä¾ĭ +æijĺ è¦ģ +å´Ľ èµ· +æѤ åĪ» +æľī æľºä¼ļ +æĿ¡ 款 +é¢Ĩ导 å°ıç»Ħ +çļĦ 身ä½ĵ +åįķ ä¸Ģ +央 è¡Į +ä¸įæĸŃ æıIJé«ĺ +ä»·åĢ¼ è§Ĥ +èĬ ½ +èIJ į +æ³ķå¾ĭ æ³ķè§Ħ +ä¸į éĶĪ +ä¸įéĶĪ éĴ¢ +åĩº äºİ +èĻļ æĭŁ +æį® æĤī +çĥ¦ æģ¼ +åħ¨ æĸ°çļĦ +æī« æıı +çĻ» éĻĨ +èīºæľ¯ 家 +çļĦ é£Łçī© +çļĦ åŃĺåľ¨ +客 åİħ +æĪij们 å°± +æŁ¥çľĭ æĽ´å¤ļ +è¯Ħ 审 +å¸Ĥ åł´ +è¬ Ľ +å·¨ 头 +ä¸ŃåĽ½ ç»ıæµİ +äºĨ èĩªå·±çļĦ +åĨ³ è®® +çĽijçĿ£ 管çIJĨ +æĬķ 票 +åĨį 度 +è¡Į çĤº +注 åħ¥ +ä½ľä¸º ä¸Ģ个 +æ¯ı个人 éĥ½ +åįķ åħĥ +è¦ģ çŁ¥éģĵ +被 称为 +ä¹ĭ éĻħ +解 éĻ¤ +ä¸ ¸ +æº « +ä¸ī æĺŁ +é²ľ æĺİ +ä¹Ł éĥ½ +æŶ æľº +åĩº æīĭ +æĥħ å½¢ +åķĨ è´¸ +éĢī 举 +对 èĩªå·± +çĶŁ åĬ¨ +åħĭ æľį +个 ä½ĵ +èĭ ij +ç¨ ± +大 åݦ +æĺ¯ å¯¹ +åĪ© æģ¯ +è¿IJåĬ¨ åijĺ +åĮĸ 解 +åīį 沿 +æĦŁ æģ© +æĢ» ä¹ĭ +é«ĺæĸ° æĬĢæľ¯ +åĿĩ 为 +åħ¨ åĮº +æ°Ķ æ°Ľ +åı¯ä»¥è¯´ æĺ¯ +ä½ı 宿 +åħļåijĺ å¹²éĥ¨ +åĹ ¯ +è·µ è¡Į +çļĦ ä¸ĵä¸ļ +èĢĥ éªĮ +èķ ¾ +åħ¬ åŃIJ +çļĦ çĬ¶æĢģ +æ½® æµģ +ä¿¡ æīĺ +è´ ¼ +åIJĦ æĸ¹ +æķij åĬ© +éĿŀ常 çļĦ +æ¡¥ æ¢ģ +åħ¬ æĸ¤ +ä¼¼ çļĦ +çľĭ 好 +å±Ģ éĥ¨ +å®ī éĿĻ +éħį 件 +常 è§Ħ +å¼Ģ 车 +第äºĮ 次 +ä¸Ĭ 级 +åıĤ èµĽ +家 å±ŀ +强 åĬ¿ +åľ¨ ä»ĸ +åIJij åīį +ä¹ĭ åľ° +éĥ ¡ +è¡Į ç¨ĭ +èѦ åijĬ +è§Ħå®ļ çļĦ +åķĨ åŁİ +äºĶ 大 +æķĻ 室 +åįģ 足 +æīĢ以 åľ¨ +å°Ĩ 继ç»Ń +çŃī æĸ¹å¼ı +家 ä¼ģä¸ļ +交 ä»ĺ +çĤ¹ è¯Ħ +ç»ĵ ç®Ĺ +ä¹Ł åı¯ +å¤ĸ æ±ĩ +è¿Ļç§į æĥħåĨµ +æİĪ äºĪ +å¸ĥ ç½® +æĪIJç«ĭ äºİ +é¢Ħ èѦ +管çIJĨ 人åijĺ +å©ļ 礼 +ç»ĵæĿŁ åIJİ +åħ¥ éĢī +æĹł æ¯Ķ +åĴĮ åıijå±ķ +çĻ½ éħĴ +çİ© åħ· +ä¸ĩ ç¾İåħĥ +çļĦ æĪIJ绩 +æĭį çħ§ +èĢĥèĻij åĪ° +ä¼ģä¸ļ åıijå±ķ +äºĨ 个 +çĶŁ æ°Ķ +çļĦ 女人 +äºĶ åįģ +çĪ· çĪ· +纽 约 +éĥ½ 被 +ä¸Ĭ 课 +çĽ ¡ +ä¼łç»Ł æĸĩåĮĸ +æ½ľ åľ¨ +åıij å°Ħ +ä¸Ģ 身 +éĺ² å®Ī +åĪ ® +é¢ĺ 缮 +åľ¨ åĨħçļĦ +ç¾İ 好çļĦ +è¿ĻéĩĮ çļĦ +ä¸Ģ ä¸Ŀ +人 åĿĩ +åĢ¡ 导 +身 åIJİ +æī© å±ķ +大 éŨ +å°± 被 +该 é¡¹çĽ® +æŀ¶ æŀĦ +ä¸Ģ åı£ +ä¿¡æģ¯ æĬĢæľ¯ +å¼Ģ ä¸ļ +æĶ¶ åıĸ +ç½ij 页 +æĶ¯ æı´ +å°ģ éĹŃ +å¡ij éĢł +大 èĥĨ +å¿«éĢŁ åıijå±ķ +çľĭ ä¼¼ +æ¸ Ŀ +è¿Ļæł· ä¸Ģ个 +模 åĿĹ +注æĦı åĪ° +çł´ 解 +èĩª ä»İ +åijµ åijµ +ä¹ĭ å¾Į +ä¹ĭ æĹħ +è·Ł æĪij +æ³ķ 人 +æİĴè¡Į æ¦ľ +åĿļ å®Ī +好 å¤Ħ +çŁ³ 头 +并 å°Ĩ +èĪ ± +æŃ ĩ +两 岸 +å¤ļ ä¹ħ +象 å¾ģ +个æĢ§ åĮĸ +çļĦ è§Ĵ度 +å¸ Ĩ +ç¦ı å·ŀ +æŁ¥ å¤Ħ +两 åĽ½ +åIJ¸å¼ķ äºĨ +é¦ĸ å¸Ń +大 åĵ¥ +é¤ Ĭ +涨 å¹ħ +éĢī çĶ¨ +許 å¤ļ +èIJ½ æĪ· +åĵĪ å°Ķ +åĵĪå°Ķ 滨 +åģļ ä»Ģä¹Ī +以 åħį +é¾ į +æĹł éľĢ +åĪ°åºķ æĺ¯ +æĢ ¡ +åijĬè¯ī ä½ł +éĺ² æ°´ +è¿Ļ æŶåĢĻ +欢 ä¹IJ +转 åIJij +è¿Ļ个 åľ°åĽ¾ +åħ¥ é©» +èįī åİŁ +æĹ¶ä»£ çļĦ +åıĺ åĬ¨ +åĬłå¼º 对 +åģ¶ å°Ķ +å®Ī æĬ¤ +æ°Ķ 温 +人 éĹ´ +æľĿ é²ľ +ç»ı è´¹ +åĽŃ æŀĹ +å·¥ åľ° +è§Ħ æł¼ +åĩł åįģ +è¯ķ åĽ¾ +å¦ ĥ +éĤ£ æŶåĢĻ +å¼ĺ æī¬ +ä¸ļ çķĮ +çļĦ éĢŁåº¦ +ä¼ļ ä¸įä¼ļ +èIJ¥ æĶ¶ +å°ıå¾® ä¼ģä¸ļ +çľĭ è¿ĩ +æĬĬ ä»ĸ +éģµ 循 +è¿Ļ è¾¹ +没æľī 人 +å£ ¶ +æ¹ĸ åįĹçľģ +æŀģ åħ¶ +çļĦ人 çĶŁ +ä»ĸ è¿ĺ +转åĮĸ 为 +èµ° è¿ĩ +æĬ± çĿĢ +çīĽ 奶 +ä¸ĩ 亩 +å¿ĥ æĢģ +æĹ¥å¸¸ çĶŁæ´» +ä½ĵ æ£Ģ +æĻ ĥ +çŃī é¢ĨåŁŁ +æĩī 該 +åı¯ä»¥ çľĭåĪ° +æī¾ ä¸įåĪ° +èĢģ å¹´ +æĬĬ æĪij +积 åĪĨ +梳 çIJĨ +ç» ³ +çļĦ æĶ¿æ²» +å¸Ŀ åĽ½ +éĻª ä¼´ +æ´Ľ éĺ³ +åħ¬ æŃ£ +å¼Ģ åı£ +çī¹èī² çļĦ +åĽ° å¢ĥ +ä¸Ĭ æľī +ç«ĭ ä½ĵ +æīĵ å·¥ +åķ¤ éħĴ +åľ¨ éĤ£éĩĮ +éĤ£ è¾¹ +个 åĪ« +ä¸Ģå®ļ æĺ¯ +çļĦéĩįè¦ģ æĢ§ +主 å¼ł +åĴĮ æľįåĬ¡ +ä¸Ĭ ç½ij +è¡¥ åĬ© +åıª éľĢ +å¼ ¦ +éģ ® +åĬĽ äºī +度 è¿ĩ +èij ¬ +é¡¿ æŶ +éĦ ī +纺 ç»ĩ +åľ° åĿĹ +ä¿¡çĶ¨ åį¡ +ç½ļ 款 +åijĬè¯ī æĪij +éĽ Ļ +书 çĶ» +è¨Ń è¨Ī +æĢ» ä¼ļ +åĪ¤ åĨ³ +ä¿¡ èªī +个 èĤ¡ +å¹³ 常 +æĢİ éº¼ +ä½ĵ çİ°åľ¨ +é»Ħ æ²³ +åĽĽå·Ŀ çľģ +羣 缸 +åIJĦ项 å·¥ä½ľ +åĬ¨ åijĺ +å³° ä¼ļ +ä¸Ģ æľŁ +æľī ä¸Ģå®ļçļĦ +é«ĺ度 éĩįè§Ĩ +ç¹ģ èį£ +åıijçİ° äºĨ +ç½ij 红 +æīĭ æ³ķ +家 åĽŃ +仪 åĻ¨ +è¾ĥ ä½İ +çļĦ å®īåħ¨ +æ¡ IJ +ä»ĺ 款 +æĬij åĪ¶ +åįĵ è¶Ĭ +æŃ£ éĿ¢ +åĵ ij +强 åĪ¶ +ä»Ĭ天 çļĦ +æĪĺ èĥľ +楼 å¸Ĥ +æĭ¿ ä¸ĭ +é¢ľ åĢ¼ +举 éĥ¨ +çłĶ åĪ¶ +çļĦ æĪĺçķ¥ +åľ¨ ä¸Ģ个 +ä¸ī 人 +å®Į äºĨ +æĸ° æĬĢæľ¯ +ç»ıæµİ æķĪçĽĬ +å¯Į æľī +æ¾³ æ´² +åĬ© çIJĨ +é¢Ĩ åıĸ +è° Ń +çĩĥ çĥ§ +ç´ł åħ» +éĤĦ æľī +è¿Ľ èĢĮ +ä»Ģä¹Ī æĺ¯ +çłĶ究 ä¸Ńå¿ĥ +éĢĤ çĶ¨äºİ +æİ¥ æĶ¶ +失 æľĽ +äºĮ 级 +éĹ´ çļĦ +åİŁ æłĩé¢ĺ +èªį çĤº +æį ¡ +对 çĿĢ +对 éĿ¢ +ä¸Ń åİŁ +éĵ ĥ +çĶŁäº§ çļĦ +åıijå¸ĥ ä¼ļ +士 åħµ +è¿Ļ åı¥è¯Ŀ +ç¼´ 纳 +ä¸Ģ个 个 +åѸ çĶŁ +çĸij éĹ® +交 èѦ +示èĮĥ åĮº +天 使 +åľ¨ ä¸Ĭæµ· +åIJĮ æĻĤ +è½» æĺĵ +åĶ¯ä¸Ģ çļĦ +çĥŃ éĹ¹ +ä¹IJ è§Ĥ +çļĦ 身份 +åĸĦ äºİ +大 åİħ +èĤ¯å®ļ æĺ¯ +éĺ² çģ« +å¤ĸ åĩº +æį® 说 +é¡¹çĽ® çļĦ +ä¸Ģ åı° +èĻļ åģĩ +ä¸Ģ ç¬Ķ +ç«ĭ æ³ķ +严 èĤĥ +æī¿ åĬŀ +åįģ åĩł +çļĦ 空éĹ´ +æľ¬ ç½ijç«Ļ +åģļ å¾Ĺ +ä¿Ŀ 温 +æľĪ åĪĿ +åľ¨ ç½ijä¸Ĭ +åIJĦ æĸ¹éĿ¢ +ä¸ī 天 +交æĺĵ æīĢ +解 æŀIJ +åħļ ä¸Ń央 +è¿Ľ åĩºåı£ +åĴĮ 社ä¼ļ +次 æķ° +ä¹ĭ 家 +ç»´ 度 +æ´¾åĩº æīĢ +产çĶŁ äºĨ +带 æľī +å¾Ī 强 +æľīäºĽ 人 +å¹´ åIJİ +äºĨ 许å¤ļ +å¯Ĩ 度 +åѦ æľŁ +çıł æµ· +æľĢå¤ļ çļĦ +è¾¹ ç¼ĺ +容 éĩı +第äºĮ 个 +ä¸Ģ缴 æĺ¯ +ä¸į ç¦ģ +æŃ ² +ä»ĭç»į äºĨ +ä¼ĺ éĽħ +æ¯Ķ è¼ĥ +èģĮ ä½į +温 æŁĶ +æľī éĴ± +æľĢ é«ĺçļĦ +åįļè§Ī ä¼ļ +ä¸į æĪIJ +éĶĻ äºĨ +è¯ģ çĽij +è¯ģçĽij ä¼ļ +æĪIJ 人 +åĿĩ åĮĢ +æľī åĪ© +è¶Ĭ åįĹ +æīĵ äºĨ +好 åIJĥ +ç³» çµ± +è·Ł éļı +çļĦ åľ°ä½į +æŃ£ å¦Ĥ +ç¨į å¾® +åį° åıij +åĪĽ ç«ĭ +é£İ åħī +å°Ĩ æĪIJ为 +ä¸į é«ĺ +é¢ij ç¹ģ +设 æľī +ä¼ ŀ +æĭĨ éĻ¤ +å½± åĥı +æ¸Ĺ éĢı +å¹´ å¼Ģå§ĭ +ç½ij æĺĵ +è¦ģ åģļ +çĶµåĬ¨ 车 +羣 å¿ĥ +æµ· åĨĽ +ä¼ł æĿ¥ +å·® åĪ« +è°¨ æħİ +çĥŁ åı° +åįĥ å¹´ +è¯ģ å®ŀ +çIJ ª +çļĦ åħ·ä½ĵ +åĪ° å¤Ħ +ä¸į å®ľ +èľ Ģ +èĥ½åĬĽ åĴĮ +çīº çī² +çļĦ éĴ± +大 éĺŁ +é¦ĸ è¦ģ +ä¸į æĦ¿ +çİ« çij° +人æ°ij ç½ij +è¿ĺæĺ¯ è¦ģ +åĽĽ å¹´ +æįŁ 伤 +çļĦ åģļæ³ķ +éĿ Ī +è¡Ķ æİ¥ +åIJĪ æĪIJ +没 人 +éŨ æ§Ľ +ä¿¡ è´· +çļĦ 缸åħ³ +举 é£İ +社 ä¿Ŀ +ä¸ĭ 游 +åĿĹ éĴ± +è¿ĩ åIJİ +çļĦ åºĶçĶ¨ +é¥ ¶ +é¢ģ åıij +ä¸Ģ å¤Ħ +åįİ å¤ı +为 ä¼ģä¸ļ +åıª ä¼ļ +ä¾µ 害 +çļĦ åĬŁèĥ½ +åѸ ç¿Ĵ +ä¸Ńåįİ æ°ijæĹı +åıijå¸ĥ äºĨ +è¿İ æİ¥ +æĪij èĩªå·± +è¿ĺ éľĢè¦ģ +太éĺ³ èĥ½ +åİ» ä¸ĸ +æĺ¯ ä½ł +åIJĪ åĬĽ +ç»ĺ çĶ» +åı° åĮĹ +çĿ£ ä¿ĥ +åĮĹ éĥ¨ +æľī å¤ļå°ij +å¾Ī éĩįè¦ģ +åĪĴ åĪĨ +åı· 线 +æĶ¾ 大 +ä¼ļ 被 +èİ· å¥ĸ +ä¹ĭ åĨħ +失 åİ»äºĨ +çݩ家 们 +éĩĩ éĽĨ +å£ ¹ +家 ä¼Ļ +çĻ½ 天 +åĽłä¸º ä»ĸ +社ä¼ļ æ²»çIJĨ +å¼Ģ åĪĽ +çĶµ ç¼Ĩ +æĸ° ä¸Ģ代 +并 è´Ń +å°± å·²ç»ı +çļĦ 社ä¼ļ +éĻ¤ éĿŀ +åı¯ä»¥ çĶ¨ +å© ī +æ¯Ķè¾ĥ 好 +å®ŀ ä¸ļ +åĪĽ åĬŀ +æıIJ èµ· +é» ĥ +ä½ı åľ¨ +å¸Ĥ æĶ¿ +éĿ¢ä¸´ çļĦ +èĥ½ åľ¨ +çŁŃ çŁŃ +羣 人 +æĺİ æĺİ +èµĦ åĬ© +çļĦ ä¸įåIJĮ +å°ı æľĭåıĭ +é¢ĺ æĿIJ +ç¾İ åij³ +æĺŁ åº§ +ä¸į ä¸Ģæł·çļĦ +çľĭ ä¸Ĭåİ» +ä¸Ģ æł¹ +广 å·ŀå¸Ĥ +åıijçĶŁ çļĦ +é«ĺ ç§ijæĬĢ +ä¸Ģ è¾ĪåŃIJ +交 åıī +ä½ĵç³» 建设 +åĽłä¸º æĪij +çıį æĥľ +ä¸Ĭ åѦ +æĪĺ æľ¯ +æѤ ç±» +交 å¾Ģ +æĮī æij© +人们 çļĦ +åħ¶ 實 +åİŁ æĿIJæĸĻ +渴 æľĽ +缸 å¤Ħ +å¾® å¾® +æ® · +ä¹ĺ åĿIJ +å¼Ģå±ķ äºĨ +é«ĺ åĵģè´¨ +æĹłäºº æľº +ä¸įæĺ¯ å¾Ī +çļĦ æĬķèµĦ +èĬĤ çľģ +èĩ ī +ç²¾ éĢī +çļĦ æłĩåĩĨ +åįĹ éĥ¨ +认è¯Ĩ åĪ° +å¹³ éĿĻ +èĹ ¥ +æī« é»ij +æī«é»ij éĻ¤ +æī«é»ijéĻ¤ æģ¶ +éĢĻ 種 +建çŃij éĿ¢ç§¯ +ç¡® ç«ĭ +管çIJĨ åĬŀæ³ķ +æĦı å¿Ĺ +ä¸ ¨ +让 åŃ©åŃIJ +æķij çģ¾ +å½ĵ ä»Ĭ +çģ« çģ¾ +åIJĦ éĥ¨éŨ +ä¾µ çĬ¯ +æ¯ı åij¨ +æı ½ +ä¸Ģ次 æĢ§ +åħ¶ä»ĸ 人 +éĶĻ è¿ĩ +ä¸İ åħ¶ +åĭĩ æ°Ķ +çĩĥ æ°Ķ +é¦ĸ å±Ĭ +æľį 饰 +ç² ¥ +å®Į æ¯ķ +å°± æĬĬ +åĬŀäºĭ å¤Ħ +ä¸Ģä¼ļ åĦ¿ +离 ä¸įå¼Ģ +å¦Ĥæŀľ æĤ¨ +ä»ĵ åºĵ +导 å¸Ī +åIJĪéĢĤ çļĦ +毫 ç±³ +å®īåħ¨ æĢ§ +ä¾Ŀ çħ§ +产ä¸ļ åĮĸ +ä½ł çľĭ +羣çļĦ å¾Ī +åѤ çĭ¬ +éĺ² å¾¡ +å¾Ī ç®Ģåįķ +é£İ æ°´ +ä½Ĩ ä¹Ł +æݨ åĩºäºĨ +æ°ijèIJ¥ ä¼ģä¸ļ +çłģ 头 +å¤įæĿĤ çļĦ +ç»ĦæĪIJ éĥ¨åĪĨ +åħħ满 äºĨ +è¿ij åĩłå¹´ +çľģ æĶ¿åºľ +æľī å¿ħè¦ģ +éĻ ³ +ä¹ĭ ç±» +ä¹ĭç±» çļĦ +æĢ§ ä»· +æĢ§ä»· æ¯Ķ +åķĨ åºĹ +å¸Ĥ åĢ¼ +人æīį åŁ¹åħ» +æ·± åıĹ +管çIJĨ å±Ģ +æģIJ æĥ§ +ä»ħ æľī +æĬµ è¾¾ +æµ· åħ³ +èµĭ äºĪ +äºĭ åĦ¿ +ä»· éĴ± +æīĭ ä¸Ĭ +èĩª å¾ĭ +åħ³ çĪ± +享 æľī +éģĹ æĨ¾ +å¾Īå¿« å°± +æĽ´ å¿« +æłĩ è¯Ĩ +åºĨ ç¥Ŀ +ä¹Ł 好 +ä¸į æĺĵ +æĪij å¾Ī +æĶ¹éĿ© åıijå±ķ +å¤ĸ åľ° +æĬµ æĬ¼ +è¯Ĺ 人 +åİķ æīĢ +æĸ° åªĴä½ĵ +èĸ Ľ +è°Ī è¯Ŀ +ä¸Ģå®ļ ç¨ĭ度 +èµ° åľ¨ +æľĢ 强 +åĬŁ çİĩ +åħ± è¯Ĩ +大 æ¡¥ +ä¸ĭ æĸ¹ +å¤ĸ èµĦ +ç¢ ± +å·¡ è§Ĩ +æ¹ĸåĮĹ çľģ +个 çĻ¾åĪĨ +个çĻ¾åĪĨ çĤ¹ +çļĦ 责任 +çļĦ åĵģçīĮ +åĬ© æݨ +åĪĽéĢł äºĨ +ä»» èģĮ +å¿« æį· +æĿij åºĦ +åİ» çľĭ +æīį èĥ½å¤Ł +å± ¤ +æĪij 家 +æĺ¯ä¸Ģ 款 +ç¾ ħ +åĨ° éĽª +æŀģ 大 +çģ¯ åħī +éĨ ĭ +ä¸İ åħ¶ä»ĸ +æıIJåĩº çļĦ +éĿł è¿ij +è°ĥ åĬ¨ +å°½ åı¯èĥ½ +åıij åĬĽ +ç»Ļ 她 +éĢĤ éĩı +è·¨ åĽ½ +åħĪ è¡Į +æĸ° æĿIJæĸĻ +ä½ľ äºĨ +满 äºĨ +ä¸į 满 +çļĦçľ¼ çĿĽ +çľĭ å¾Ĺ +è¿Ļ ä¸Ģ次 +é½IJ åħ¨ +çļĦä¸Ģ éĥ¨åĪĨ +ä¸ Ļ +æ¸ħ æĸ° +說 æĺİ +身边 çļĦ +æīĢæľī 人 +å½° æĺ¾ +è± ¹ +åį ¿ +è¿IJ 转 +æĮĩ å¼ķ +å¸Ĥ åħ¬å®īå±Ģ +åıĤ å±ķ +ä¹ĭ æŶ +éĩijèŀį æľįåĬ¡ +èµĦæľ¬ å¸Ĥåľº +èĥ½ 让 +å¿ĺ äºĨ +天 åłĤ +æ¯Ķå¦Ĥ 说 +éĬĢ è¡Į +èĽĭ ç³ķ +çĶ © +æł¸ å®ŀ +æĻ® 京 +ä¼ĺ ç¾İ +åı£ èħĶ +漫 çĶ» +çľ¼ éĩĮ +äºĨ ä¸ĭæĿ¥ +æĪij们 ä¹Ł +ä¾ į +为 ä¸Ńå¿ĥ +å¥ĩ 迹 +éĿĴ çĿIJ +æĪªèĩ³ 缮åīį +åĩº ä¾Ĩ +æĢ» åħ¬åı¸ +å¼¥ è¡¥ +ç®Ĺ æ³ķ +å·¥ä½ľ 室 +æīĢ以 æĪij +æ°´ åĪĨ +æīĢ å±ŀ +ä¸į 说 +ä½Ĩæĺ¯ åľ¨ +è¦ģ åİ» +åĪĽä¸ļ èĢħ +ä¸į æ¸ħæ¥ļ +åĽĽ åij¨ +æĺ¯ ä»İ +çļĦ æł¹æľ¬ +çģ ¶ +æ¯Ľ æ³½ +æ¯Ľæ³½ 举 +æµ· åı£ +åĽĽ åįģ +ä¹Ł 被 +èģ · +ä¸Ģ æīĭ +绩 æķĪ +çļĦ çĶ·äºº +书 ç±į +ä¸Ģ èĦ¸ +大 äºİ +鼶 éĥ¨ä»¶ +åħ³ æĢĢ +å¹³ ç±³ +æļ´ éľ² +å¾Ĺ å¤ļ +ä¸ī 级 +æľ¬ åij¨ +两 èĢħ +对 ä¸ŃåĽ½ +åıª è§ģ +欧 ç¾İ +å¦Ĥæŀľ æľī +å·²ç»ı æĺ¯ +çľĭ å®Į +çģ« éĶħ +èµ IJ +ä¸Ģ éģį +æĦŁ åĨĴ +ç»ĵ å±Ģ +ä»ĵ åĤ¨ +å®ŀ åľ° +åī¯æĢ» ç»ıçIJĨ +ä¹Łä¸į çŁ¥éģĵ +碰 åĪ° +åIJĪ 计 +客æĪ· çļĦ +ç½Ĺ 马 +æĦī å¿« +é£ Ľ +çĥŃ çĥĪ +伦 æķ¦ +åĮ» ä¿Ŀ +éĺ¿éĩĮ å·´å·´ +åĨį 说 +为 åŁºç¡Ģ +çĶŁäº§ ç»ıèIJ¥ +è¿ĻäºĽ 人 +åĪĹ è½¦ +æ²³åĮĹ çľģ +è¿Ļ 段 +æ´»åĬ¨ ä¸Ń +å© · +çĶŁ çIJĨ +ä¸ŃåĽ½ 人æ°ij +éĦ Ĥ +åIJ¬ åıĸ +å¤į ä¹ł +æľī çĽĬ +æĶ¶ æĭ¾ +å¾Ī åı¯èĥ½ +ç½ij绾 游æĪı +们 çļĦ +èµĭ èĥ½ +éļ¾ å¾Ĺ +åĪĨ æīĭ +羣 è¯ļ +åħ¬åı¸ åľ¨ +åĿĩ è¡¡ +åı£ åij³ +çīµ 头 +ä¸ĢèĪ¬ çļĦ +轿 车 +çŃī äºİ +æ²ī é»ĺ +æĪij éĥ½ +å°ı ç¨ĭåºı +ä¸Ģ åī¯ +æī¿ è½½ +åľ° è´¨ +çķĮ éĿ¢ +çĶµ æľº +çĦ¦ èĻij +éĶĢåĶ® é¢Ŀ +æĸ° 车 +ä¸Ĭ 游 +主 æ¼Ķ +éļIJ ç§ģ +åıijå±ķ æĪĺçķ¥ +çļĦ åĬªåĬĽ +å¼Ģ åħ³ +解åĨ³ éĹ®é¢ĺ +çĿ£ 导 +对 æĬĹ +å¾Īå¤ļ 人éĥ½ +æĹł æķĪ +产åĵģ è´¨éĩı +å®ī å¿ĥ +åįİ äºº +ä¸į 符åIJĪ +èĩª 家 +éĺµ å®¹ +çļĦ åIJĦç§į +çļĦ çIJĨ念 +çļĦ æĸĩåĮĸ +为 èĩªå·± +å±± æ°´ +游 æ³³ +éľĩ èį¡ +çĶŁæ´» æĸ¹å¼ı +è¿ľ 离 +çŁ³ åĮĸ +æѤ äºĭ +æĺ¯ çľŁçļĦ +çļĦ æ¯Ķä¾ĭ +çĶ¨ çĶµ +奥è¿IJ ä¼ļ +ä¿Ŀ å®ī +èĽĭçĻ½ è´¨ +çļĦ å¿ĥçIJĨ +å· « +åı· çłģ +æ°Ķ ä½ĵ +åıij æĶ¹ +åıijæĶ¹ å§Ķ +åĮ» å¸Ī +æ¶Ĥ æĸĻ +æĺ Ĭ +å¸Ĥ 级 +ä¸ĸçķĮ çļĦ +åĪĨåĪ« æĺ¯ +çł´ 产 +ä¸Ģ æĿ¯ +æĭī å¼Ģ +å¹³ åĩ¡ +çļĦ åıijçĶŁ +åĬ¨ æīĭ +ä¸Ģ缴 以æĿ¥ +æīĭ å·¥ +éĩĮéĿ¢ çļĦ +æĹł åħ³ +ä»ĭ åħ¥ +èµ° ä¸Ĭ +å°±æĺ¯ è¦ģ +å¹´ éĹ´ +åĩº çı¾ +å½± éŁ¿ +å¹ħ 度 +éĽ ģ +éģĵ åħ· +缮çļĦ åľ° +åIJİ èĢħ +ä¸Ĭ æ¼Ķ +äºĨ åĩł +æ®ĭçĸ¾ 人 +å¿Ļ ç¢Į +æĺ¯åIJ¦ æľī +并 对 +ä¼ļ 导èĩ´ +æ°´ åºĵ +ç»Ĩ èĩ´ +åIJİ æĤĶ +å¿ĥ æĢĿ +åģļ äºĭ +åİĤ æĪ¿ +çĿ ¿ +è¿IJèIJ¥ åķĨ +头 éĥ¨ +çļĦ è§Ĵèī² +æĺ¯ ä»ĸ +æĹ¢ æľī +å°ıæŶ åĢĻ +强 åĬ² +主 æĴŃ +åħ¨åĽ½ åIJĦåľ° +æį ı +æįŁ åĿı +åķĨ ä¼ļ +ä¿Ŀ ç½Ĺ +çľģ å¸Ĥ +éļ§ éģĵ +æľī ä¸įå°ij +è¦ģ åľ¨ +建设 é¡¹çĽ® +ç³ĸ å°¿ +ç³ĸå°¿ çĹħ +æĿ¡ä»¶ ä¸ĭ +ä¼ĺè´¨ çļĦ +é¦ĸ åıij +å½ĵæŶ çļĦ +丰 çĶ° +大 çĽĺ +缸 继 +å®ģ å¤ı +åħ¥ ä½ı +æĪij è¿ĺ +åħĭ æĸ¯ +å®ļ ä»· +å¹³æĸ¹ åħ¬éĩĮ +çļĦ çŁ¥è¯Ĩ +æĪij们 ä¼ļ +åħĥ å®Ŀ +ä½ĵ éĩį +è³ £ +对 æĪij们 +çŁ³ 家 +çŁ³å®¶ åºĦ +ç²¾ åįİ +å½¢ çĬ¶ +åıĹ åĪ°äºĨ +ä¿® 订 +ç¾İ åľĭ +é«ĺ æ¸ħ +çľ¼ éķľ +è§īå¾Ĺ èĩªå·± +带 ç»Ļ +åĶ® ä»· +éŨ 票 +åŃķ å¦ĩ +çĶµè§Ĩ åı° +åıij ä½ľ +çļĦ åij³éģĵ +éķ¿ è¿ľ +åħ¬åħ± æľįåĬ¡ +æŃ£å¸¸ çļĦ +æľī è¿ĩ +é£İ æĥħ +æ¯Ķ éĩį +åIJ » +管çIJĨ å·¥ä½ľ +综åIJĪ æĢ§ +å·² 被 +说 èµ· +æİĴ æ°´ +ä¸įæĸŃ åľ° +æĥħ æĢĢ +è¾ĵ éĢģ +è¿ĩ æķı +çļĦ åı¯èĥ½æĢ§ +æľį çĶ¨ +æľī 许å¤ļ +å§Ķ åī¯ä¹¦è®° +åĮĸå¦Ĩ åĵģ +æļĤ åģľ +æĬķèµĦ 人 +çıŃ çº§ +说 çĿĢ +åįĹ åĮĹ +åĪĨ è¡Į +çıł å®Ŀ +å¯ ¶ +å¢ŀ å¤ļ +被 åĬ¨ +çī¹æ®Ĭ çļĦ +éĹľ ä¿Ĥ +çļĦ èĦ¸ +æĥ Ł +ä¸į ä¸Ģå®ļ +ç¶ Ń +çģ« çĪĨ +秣 éĩij +çŀ § +éĩį 建 +è· ª +ä¸Ģ 種 +çļĦ åIJĪä½ľ +å®ī æħ° +ä»į æĺ¯ +ä¸ĵä¸ļ åĮĸ +è°ĥ 解 +ä¸į 妨 +éĢĻ æĺ¯ +å¿ħ éłĪ +ä¼Ĭ æľĹ +å¾Ĺ äºĨ +æľįåĬ¡ å¹³åı° +å§ ¬ +åħĪ éĶĭ +çİĭ åŃIJ +çļĦä¸Ģ åĪĩ +æĢ» çIJĨ +åĵ ¼ +çª ij +çļĦå¿ĥ æĥħ +çļĦ éĩį大 +çij Ł +ä¸Ģ ç¬ij +åıijå±ķ ä¸Ń +åģ¥åº· åıijå±ķ +åĵģçīĮ çļĦ +ç¦ ® +ä½Ļ 人 +ä»Ĭå¹´ 以æĿ¥ +æķ° çłģ +çŃ¾ è¯ģ +åİ» æī¾ +åŁºéĩij ä¼ļ +æĬ± æĢ¨ +æŃ£ å½ĵ +çıŃåŃIJ æĪIJåijĺ +ä¸į åIJĪæł¼ +åĪ¶ å®ļäºĨ +ç¼ĵ æħ¢ +åĪ¶ 约 +æłı 缮 +å¸Ĥåľº ç»ıæµİ +ç»ĦæĪIJ çļĦ +严 å³» +æĹ¥ 讯 +ä¸ĢçĤ¹ çĤ¹ +æĺ¯ æĢİä¹Ī +çļĦ çħ§çīĩ +éĺ» æŃ¢ +模 ç³Ĭ +ç¼ ¸ +éģķ åıį +æIJ¬ è¿ģ +éĩij éĴ± +å½ ¬ +ä¸į å®ī +æĪĺçķ¥ åIJĪä½ľ +å¡« åĨĻ +讲 究 +åħħåĪĨ åĪ©çĶ¨ +èĥ½ å¤ł +èij¡èIJĦ éħĴ +éĩĩçĶ¨ äºĨ +åľ¨ ä»Ĭå¹´ +ä¸Ńå°ı åѦ +åľ¨ æĦı +çļĦ åİĭåĬĽ +ä¸į 幸 +åĪ¶ èį¯ +åı¯ä»¥ 让 +被 è¯Ħ为 +ç»Ĩ èıĮ +æĪı åī§ +åįĬ 导 +åįĬ导 ä½ĵ +è§Ĩ è§Ĵ +åĸľ æŃ¡ +å¾ģ æĶ¶ +è°ĭ åĪĴ +æŀģ 大çļĦ +çĤ¹ èµŀ +è®°èĢħ ä»İ +两 åIJį +èĩª åĬ© +èµ· æŃ¥ +æĬ¤ 士 +å®Ŀ 马 +太 åŃIJ +å°ıå°ı çļĦ +温 æ³ī +åĩºç§Ł 车 +秣 æĪ¿ +两 家 +éľĩ æĴ¼ +ç§ī æī¿ +ä¸Ģ件 äºĭ +çĥĪ 士 +å®ĺ åħµ +转 身 +ä¹IJ åĽŃ +çĻĮ çĹĩ +模 èĮĥ +æĦ £ +è¿ĩåİ» çļĦ +代 ä»· +çļĦ æ¦Ĥ念 +åĩł çĻ¾ +è´µ éĺ³ +æĭħ 忧 +éĢĤ å®ľ +çݯå¢ĥ ä¿ĿæĬ¤ +çĥ « +ä½ł æĥ³ +æѤ åIJİ +ä½ł ä¹Ł +çį İ +éĻ¤ æѤ +éĻ¤æѤ ä¹ĭå¤ĸ +è°ĥ 度 +ç§ij 缮 +æīĢ说 çļĦ +åĬ ĩ +忽 è§Ĩ +ä¸ī 次 +ä¸Ģ æĹ¥ +åŀĤ 缴 +ç«ŀ æĬĢ +éĿ¢ åĮħ +大 æĪĺ +æIJº 带 +å¦Ĥæŀľ 没æľī +åħ» æĪIJ +åĩº è¡Ģ +çĪ±å¥½ èĢħ +æīĵ éĢļ +èµ· è¯ī +åijĪ çİ°åĩº +æŃĮ æīĭ +åľ¨ å¤ĸ +é¢Ĩ导 å¹²éĥ¨ +åĨ ¥ +èĪĨ 论 +æıIJ åıĸ +éĺ¿ å°Ķ +æľĽ çĿĢ +ä¸ī äºļ +è² ¡ +åĪ ·æĸ° +æĻļ æĬ¥ +è¿ĺæľī ä¸Ģ个 +åĨ° ç®± +ç½ij çĤ¹ +åĩº åħ· +强çĥĪ çļĦ +æĪij çĽ¸ä¿¡ +å¸ĮæľĽ èĥ½ +çīĻ 齿 +äºĭ å®ľ +ä¸ļåĨħ 人士 +代 æĽ¿ +åıĺ å½¢ +éĽ ² +è°ĥ æݧ +åĪĽæĸ° åĪĽä¸ļ +æĭĨ è¿ģ +æł¸ æŁ¥ +éĢ Ĺ +åħ¥ åѦ +æĦı åIJij +æı Ľ +ä¸ĭ 次 +ä¼ł è¾ĵ +ä»ĸ们 åľ¨ +èĢĮä¸Ķ è¿ĺ +æĹ¥ åľ¨ +æķĻ è®Ń +æ´» çĿĢ +çļĦ æľīæķĪ +å¤įå·¥ å¤į +å¤įå·¥å¤į 产 +æĺ¯ä¸Ģ 件 +çŃī çĿĢ +å¾ © +åĭĩ æķ¢ +éģŃ åıĹ +å¥Ķ é©° +讲 座 +说 å®Į +ç»Ļ åĩº +è° ¦ +è¯Ĭ çĸĹ +çĽ² 缮 +客 è¿IJ +å°± è¿ŀ +å¼Ģ åħĥ +å¼Ģåħĥ æ£ĭçīĮ +ä¸įæĸŃ æıIJåįĩ +çĶ¨æĪ· çļĦ +æĴ ķ +ä¾Ľ æ°´ +ç¶ĵ æ¿Ł +ä¸Ń åĮ»èį¯ +èģĶ æĥ³ +åħ¬äº¤ 车 +èĪª çıŃ +æĬĢ è¡ĵ +å¼ķèµ· çļĦ +å° ¹ +èµĦ æ·± +åĽ½èµĦ å§Ķ +èĺ Ń +é¼» åŃIJ +éĹ ½ +æİĴ éĺŁ +è§Ĥ åħī +éģĹ åĿĢ +举 京 +é¥Ń åºĹ +ä¸įæĸŃ çļĦ +å°±æĺ¯ ä¸Ģ个 +éķ¿ ä¹ħ +çļĦ è§ĤçĤ¹ +å¨ ¶ +æĪij çİ°åľ¨ +çķ ° +å¾Ĺ åĩº +å¿ħ å®ļ +ä¸į åıĹ +åıª éľĢè¦ģ +åĽ° æī° +ç§ijåѦ æĬĢæľ¯ +çīĽ èĤī +è¾ĥ é«ĺçļĦ +è·ij æŃ¥ +æ² ¾ +èı© èIJ¨ +æľĢ å¾Į +ä¿Ŀ å¯Ĩ +æ²» å®ī +éĤ ± +常 è¯Ĩ +èĦ¸ èī² +åĮĹ å¤§ +æ±ĩ èģļ +æijĨ èĦ± +é¾Ļ头 ä¼ģä¸ļ +女 åıĭ +çŃī å·¥ä½ľ +ä¸Ń ç¾İ +èģĮ åľº +èĦij è¢ĭ +åĨĻ çļĦ +饲 æĸĻ +åĬ³ åĬ¨åĬĽ +å± ¯ +æĮģ èĤ¡ +åĽ¾ åĥı +è¿ĩåİ» äºĨ +è² ¨ +è¾ ² +éĹ® æĪij +è·Ł ä½ł +çĶŁ æŃ» +审 ç¾İ +é¢Ĺ ç²Ĵ +ä¸Ń æĸ¹ +åĬł çĥŃ +æĹħè¡Į 社 +çĻ¼ çĶŁ +ä¸į åłª +åĤ · +æ¥ ł +åĬŀ æ¡Ī +æŁ Ħ +æĹ¢ æĺ¯ +å¤Ħ åĪĨ +羣å®ŀ çļĦ +æĬ¥ 纸 +å¸Ī çĪ¶ +å®īå¾½ çľģ +åī¯ 主å¸Ń +ä¹ĭ éģĵ +导 å¼¹ +åŃ¦æł¡ çļĦ +åŁİå¸Ĥ çļĦ +è°Ī åĪ° +æ¢ Ĺ +å¹³ éĿ¢ +说 ä»Ģä¹Ī +é¢ij çİĩ +éķ¿ ä¸īè§Ĵ +çļĦ åĪ©çĽĬ +é» ¨ +è±Ĩ èħIJ +å®ŀéĻħ æĥħåĨµ +æŀĹ ä¸ļ +纪æ£Ģ çĽijå¯Ł +ä½ı éĻ¢ +çļĦ æķ´ä½ĵ +åīį è¡Į +æĮ ¨ +çħ¤ çŁ¿ +åī¯æĢ» è£ģ +å°ı åIJĥ +æŀģ 端 +å©Ĩ å©Ĩ +çİ° è´§ +è¯Ĺ æŃĮ +éĴ¥ åĮĻ +缩 çŁŃ +ä½Ĩ è¿Ļ +æĸ° åĵģ +è¿Ļ 对 +çŁ¥åIJį 度 +å¿ĹæĦ¿ æľįåĬ¡ +大 å±Ģ +è¡¡ éĩı +ä½ĵçİ° äºĨ +æ¡ĥ èĬ± +åIJ¸å¼ķ åĬĽ +åł ¤ +æĵħ éķ¿ +åĴ Ĵ +缸 æľº +ä¸Ģ ç«Ļ +ä¸Ģç«Ļ å¼ı +æľĢ ç¾İ +æ°¸ ä¹ħ +çļĦ éĥ¨åĪĨ +åĪĨ å·¥ +å·¥ç¨ĭ 建设 +æIJŃ è½½ +æ°´ ä¸Ń +èĮ ¨ +çļĦ æĵįä½ľ +绣 æ²» +çķħ éĢļ +åħļçļĦ åįģ +è¼ ¸ +æ¸ ¬ +ç¾İ è§Ĥ +ä¸į åĪ© +åıį æĢĿ +éªĦ åĤ² +æłĩ çļĦ +æĿĢ 人 +éĺ¿ å§¨ +é£Ł æĿIJ +åIJĥ çļĦ +åIJİ åĨį +çŁ £ +两 侧 +æ¸ħ æ°´ +è¿Ľ çIJĥ +å¼Ģå§ĭ äºĨ +åIJ¬ äºĨ +çĦĬ æİ¥ +çŁ ® +å¨ Ł +为 人 +éĢģ ç»Ļ +åĨĴ éĻ© +æķ · +ç»Ī æŃ¢ +æīį çŁ¥éģĵ +è¿IJ æ°Ķ +éĢļ é£İ +æĥĬ 讶 +ç§ijåѦ éĻ¢ +æıIJ éĹ® +太 åİŁ +缸åIJĮ çļĦ +ä» ķ +èģ ĸ +æĥħ æ³ģ +é¢Ĩ导 人 +åĩºæĿ¥ äºĨ +沿 线 +éĻ ½ +æĦŁ 覺 +ä»į åľ¨ +æ© Ļ +约 为 +åĸĿ éħĴ +çĶ¨ èį¯ +ä¸ĭ ä¸Ģ +æ³ķ å®ĺ +顺 åºı +åģļ ä¸Ģ个 +åĭ ¢ +æŃ ª +çĶµ ç«ŀ +ä¼´ éļıçĿĢ +ä¹ĭ åĬĽ +ä¹ĭ 人 +äºij 计ç®Ĺ +åĪ«äºº çļĦ +ç§ijåѦ åıijå±ķ +第 åħ« +å¹² æī° +女 ç¥ŀ +è¿Ļæł· åģļ +å¤Ħ åľ¨ +æ°´ è´¨ +éķ¿ æĺ¥ +å¸Ĥåľº éľĢæ±Ĥ +ç»´ æĿĥ +èĢ³ æľµ +æĸĩåĮĸ çļĦ +奶 ç²ī +ä¼ł è¾¾ +æīĭæľº çīĪ +æĽ¾ åľ¨ +äºĮ æľŁ +åİŁåĽł æĺ¯ +æºIJ 头 +åıĪ èĥ½ +è£ ¸ +æĬĢæľ¯ åĪĽæĸ° +æĸĩåĮĸ æĹħ游 +åıij 票 +å¹´ 级 +ä½ł ä¸į +ä¹ĭ å¿ĥ +æķ° çĻ¾ +åIJij å¾Ģ +èĢģ 家 +åľĭ éļĽ +çļĦ é«ĺ度 +æľĿ éĺ³ +æ¸ħ éĻ¤ +èĩª æľī +书 ä¸Ń +游æĪı è£ħå¤ĩ +ä¸ĩ å¤ļ +驾驶 åijĺ +ä½ł çŁ¥éģĵ +åĽ½ åºĨ +é£Ł åłĤ +æİ¥ åı£ +æĢ» æķ° +åħ¶ä»ĸ çļĦ +çĶŁåij½ çļĦ +ä½ł åľ¨ +çļĦ 缮åħī +è¿Ļ æĸ¹éĿ¢ +éĥ½ 说 +çĸĹ æ³ķ +åĭĩ 士 +åľ¨ åħ¨çIJĥ +ä¿ĿéĻ© åħ¬åı¸ +çĿ£ æŁ¥ +åĸĦ èī¯ +表 å½° +è¹ ² +è·¯ 段 +æľĥåĵ¡ è¦ı +æľĥåĵ¡è¦ı ç¯Ħ +æĪ· åŀĭ +ä¿ĥ 使 +ä¿® 建 +é«ĺ æ°´å¹³ +åģļ åĩºäºĨ +主 åľº +è¡Į èµ° +空 çĻ½ +æľī人 说 +è¿Ļ个 ä¸ĸçķĮ +åIJį ä¹ī +å®Į ç¾İçļĦ +羡 æħķ +åıĬ åħ¶ä»ĸ +åı¯ çĶ¨ +æĭ IJ +è¾ĥ 大çļĦ +æĬĢæľ¯ åĴĮ +å°¼ äºļ +çĻ¾ è´§ +æı ī +éĢī è´Ń +éĺŁ åıĭ +ä¼ł æĦŁ +ä¼łæĦŁ åĻ¨ +åıªè¦ģ ä½ł +为ä»Ģä¹Ī è¦ģ +ä¸ĵ注 äºİ +ä½Ļ é¢Ŀ +åħ¸åŀĭ çļĦ +缮åīį å·² +欲 æľĽ +èģĶ 绾 +æµģ ä¼ł +çļĦ 家åºŃ +åı· åı¬ +çıį è´µ +ä¼Ł 大çļĦ +éī´ äºİ +è·Ł ä»ĸ +产 çī© +ä¸į å·² +è¿Ŀæ³ķ è¡Į为 +头 ä¸Ĭ +åĪĨ 解 +åı¯ä»¥ çľĭåĩº +æł¡ åĮº +åŃĹ ä½ĵ +ä¿® çĤ¼ +çĶļèĩ³ æĺ¯ +微信 åħ¬ä¼Ĺ +åıĸ 代 +èIJ¥ä¸ļ æĶ¶åħ¥ +æ½į åĿĬ +ä½ł èĥ½ +社ä¼ļ ä¿Ŀéļľ +æ¯ĶèµĽ ä¸Ń +污水 å¤ĦçIJĨ +夫 å¦ĩ +ä¸Ģ å¹ħ +沿 æµ· +åı£ æĦŁ +ä½Ĩ åį´ +å½ĵ æĹ¥ +çļĦ æľĢ大 +æ¯ı ä¸Ģä½į +没 äºĭ +çī¹ åĪ¥ +å¼Ģ åѦ +è·¯ éĿ¢ +å¿ĥçIJĨ åѦ +æĶ¾ ç½® +éĩįåºĨ å¸Ĥ +ä½ł èĩªå·± +æ¶Īè´¹èĢħ çļĦ +ä¸Ģ æ³¢ +èѦ æĥķ +åį§ 室 +注 å°Ħ +é£İ 鼨 +沿 çĿĢ +åijĬ 訴 +表 çİ°åĩº +åĽĽ æĺ¯ +åı¤ åħ¸ +æĽ´ éĩįè¦ģçļĦ +好 äºĭ +çľ¼ 泪 +æ¨ ĵ +审 åĪ¤ +碰 æĴŀ +车 ç«Ļ +è¿Ľåħ¥ äºĨ +éĽĨ åIJĪ +æł¼ å¤ĸ +宾 é¦Ĩ +æĶ¯ä»ĺ å®Ŀ +她 æĺ¯ +æĺ¯ å¦Ĥä½ķ +人 次 +çļĦ æĪIJåĬŁ +æĹł åĬĽ +æµ· æĭĶ +æĺ¥ åŃ£ +éĥ½ ä¸įä¼ļ +çŃī å¤ļç§į +ä¸Ģ个 å°ı +åģľè½¦ åľº +让 æĽ´å¤ļ +è¿Ļ çĤ¹ +æĪIJ åĵģ +éĴ ī +éģĩ è§ģ +çıŃ ä¸»ä»» +æĦı æĦ¿ +çļĦ åIJĮåѦ +游 è§Ī +åİĭ 缩 +åľ¨ ä¼łå¥ĩ +å¼¹ æĢ§ +æĹ¥ åĨħ +ç¦ı建 çľģ +è§Ĵ èIJ½ +åĪĨ å¼Ģ +ä¼ļ 让 +å¤ĸ åĽ´ +çĨŁæĤī çļĦ +çĨ Ķ +ä¸ĩ è¾Ĩ +å¤ľ éĹ´ +车 身 +ä¸Ń æľŁ +å®ĮåĸĦ çļĦ +åĵģ ç±» +åıĭ è°Ĭ +éĢīæĭ Ķ +éªij 士 +å½ ¦ +çļĦ çľĭæ³ķ +åĽ½ çİĭ +è¾£ æ¤Ĵ +åıijå¸ĥ æŶéĹ´ +åı¤ åŁİ +éļı æľº +ç« ĸ +å¼Ģ è¾Ł +ä¼Ĺ çĶŁ +没 åĬŀæ³ķ +åįĥ éĩĮ +æĿ¥æºIJ äºİ +çļĦ æĿĥåĪ© +æ¯Ķ åĪĨ +满æĦı çļĦ +ä¿® è¡Į +åĿ ł +大 æµ· +èİ ¹ +åĩº 身 +è« ĩ +åħ³ èĬĤ +åIJį 人 +éľĢè¦ģ 注æĦı +æĹ© æĻ¨ +å¤ĸ åįĸ +åıĪ è¦ģ +æ¶ī æ¡Ī +çĶ³è¯· 人 +éĻĦè¿ij çļĦ +åĬłå¿« æİ¨è¿Ľ +æĸ° å¹´ +大 è¡Ĺ +ä¸Ģ é»ŀ +èĭı å®ģ +æĤĦ æĤĦ +èĦ¾ æ°Ķ +å¸Į èħĬ +éļı åį³ +æķ¢ äºİ +å®ŀè·µ ä¸Ń +æĺ¯ æ²¡æľī +æľī趣 çļĦ +æĿ¥èĩª äºİ +è£ģ åĪ¤ +女 åŃ©åŃIJ +èĩ³ åħ³ +èĩ³åħ³ éĩįè¦ģ +æĻº åĬĽ +èµ° åĩºåİ» +çŁŃ æĿ¿ +大 åĽ½ +çļĦ 认è¯Ĩ +å¹´ å¤ľ +åĨį åĪ° +åIJĮ æł·çļĦ +å¯Ĩ å°ģ +å¤ĸ交 éĥ¨ +çĶŁ æķĪ +æĤ¨ åı¯ä»¥ +ä½ł åĢij +è¿ĩ å¹´ +å¼ ĵ +è¡Į æĿİ +æ¯Ķ èµ· +身 é«ĺ +è¿Ļ个 人 +ä¸Ń å¤ĸ +éģĵ æŃī +çĽ¯ çĿĢ +亲 åŃIJ +éĹ ¸ +çĻ½ äºij +èĦĸ åŃIJ +ä¸ĢåĪĩ éĥ½ +æ· ij +è° ľ +åģ¶ çĦ¶ +éĿł è°± +é«ĺ 管 +ä¸ĭ åıij +æĶ¾ åĪ° +ç±» åĪ« +ä¸ĭ åĪĹ +æ·· ä¹± +åIJĪæ³ķ æĿĥçĽĬ +çݯ çIJĥ +æľīæķĪ åľ° +åķĨ æĪ· +æ¹ĸ 人 +æµ· 岸 +æĬķ 产 +两 个æľĪ +éĥ½ éĿŀ常 +å¢ŀ强 äºĨ +æĿ¥ åĪ°äºĨ +åī© ä½Ļ +æĤ¨çļĦ åŃ©åŃIJ +æµģ æ°´ +æŃ£ ä¹ī +天 çĮ« +åģļ è¿ĩ +ä½ķ æŶ +æĪij åİ» +çľģ 份 +å¥ĸ éĩij +该 å¦Ĥä½ķ +ä¸ĭ çıŃ +åģ¶ åĥı +æijĨ æĶ¾ +æĸ° 模å¼ı +æĬķ è³ĩ +è·¯ åı£ +åĨľæ°ij å·¥ +大 åѸ +件 äºĭ +æł¹æľ¬ ä¸į +æµĵ 度 +æµĵ åİļ +è½® èĥİ +æĪ¿ ä¼ģ +éĿŀ常 好 +ä»İ ä¸Ń +人 æł¼ +ç¿ ģ +æŶéĹ´ åĴĮ +è¿Ļ ä¸įæĺ¯ +åĪ¸ åķĨ +æĥĬ 人 +åĻ¨ å®ĺ +åĩĨ åĪĻ +æĥħ æĻ¯ +æĽ´ é«ĺçļĦ +åѦ 家 +泡 沫 +åľ°æĸ¹ æĶ¿åºľ +å°± çŁ¥éģĵ +åij¼ åIJģ +ç»ı è´¸ +èĬ± éĴ± +æľī ä¸Ģ次 +æĦŁ æħ¨ +ä¸Ģ åįĥ +å¤ľ æĻļ +詹 å§Ĩ +詹å§Ĩ æĸ¯ +è¦ģ éĹ» +ç» Ĵ +æºIJ äºİ +çļĦ è´¨éĩı +注æĦı äºĭ项 +æħ¢ æĢ§ +稳å®ļ çļĦ +建设 åĴĮ +æĻ¯ 象 +éĩı åĮĸ +çļĦ 話 +è¯Ħ 级 +æº ľ +红 åĮħ +éĢļ éģİ +社ä¼ļ 责任 +æĸ° 产åĵģ +åĨ· éĿĻ +çľĭ ä¸įåĪ° +èģĶ éĤ¦ +éŃ Ħ +çļĦ åīįæıIJ +çļĦåīįæıIJ ä¸ĭ +è¾ĥ 好 +çļĦ æĦŁæĥħ +客æĪ· æıIJä¾Ľ +çĭ¬ èĩª +å¢ŀ æĶ¶ +æĸĩ çĮ® +æĭ¼ åij½ +管çIJĨ åĴĮ +æµģåĬ¨ æĢ§ +åħ¨ 家 +ä¸Ĭ æĸ¹ +æݨåĩº çļĦ +ä¸ī åĽ½ +ä¸Ģ个 æĺ¯ +æĸ° ä¸Ģè½® +æĸĩåĮĸ éģĹ产 +æ® º +大 æ¹¾åĮº +éĥ½ éľĢè¦ģ +çļĦ å®ŀéĻħ +ç· Ĭ +大 å¥ĸ +åħī èĬĴ +便 äºİ +çļĦ 表æĥħ +æ¼Ķ ç»İ +红 åĨĽ +å½ĵ æĪij +æ²» æĦĪ +é¢Ŀ 度 +éĿ ľ +ä»»ä½ķ 人 +è¡Ĺ 头 +çī¹ æĸ¯ +çī¹æĸ¯ æĭī +åĮ»çĸĹ æľºæŀĦ +ç»Ļ åŃ©åŃIJ +è§Ħ 磩 +è£ ľ +çļĦ 身影 +ä¸ĵ æłı +æĿ¥ 临 +ç«¥ å¹´ +å¤į èĭı +è¨ Ĥ +åŀĭ åı· +åĽ¾ æ¡Ī +ç®Ģ åİĨ +æĭ ± +èį· åħ° +ä»» æĦı +æī¿ æİ¥ +è¿Ļ æīį +客 车 +æľĿ çĿĢ +éłħ 缮 +åı° é£İ +çļĦ æĪ¿åŃIJ +éª ı +æĿ± 西 +éģĹ ä¼ł +è¶Ĭ å¤ļ +äºĨ ä»ĸçļĦ +ä¸Ĭ åij¨ +管çIJĨ åĪ¶åº¦ +失 ä¸ļ +çĶ· åıĭ +æİ¥ ç§į +å¨ģ åIJį +çĴ° å¢ĥ +åıijçĶŁ åľ¨ +个 åĽ½å®¶ +åĪĽæĸ° åıijå±ķ +æĶ¹åıĺ äºĨ +åģ¥åº· çļĦ +åĢ¼å¾Ĺ ä¸Ģ +åĢ¼å¾Ĺä¸Ģ æıIJ +åĽ¢ ä¼Ļ +åģĩ 设 +åı° ä¸Ĭ +è§ĦèĮĥ åĮĸ +éĻª åIJĮ +座 æ¤ħ +åı¯ æĢľ +åħĭæĢĿ 主ä¹ī +æ³ķå¾ĭ 责任 +ä¸Ģ é¡¿ +æĬ¬ 头 +为 éĩįçĤ¹ +è¿ľ æ´ĭ +éĢı è¿ĩ +åħ¨çIJĥ åĮĸ +趣 åij³ +票 æĪ¿ +æ¯ı 人 +åIJĦç§į åIJĦæł· +äºĨ åĩºæĿ¥ +ç»Ŀ对 æĺ¯ +ä¸ĭ å±ŀ +ä¸Ģ åıĮ +è¿Ļ åĿĹ +æĬĹ çĸ« +è¦ģ çĤ¹ +å½¢æĪIJ çļĦ +æĪij çľĭ +ä¸ĩ éĩĮ +èĢĥ çłĶ +为 åħ¶ +æ°ij 宿 +å¤ļ ä½į +大 èĩ´ +ä»ĺ è´¹ +åħ¥ æīĭ +å±ħ 家 +æīĢåľ¨ åľ° +人 身 +è¿ĩ å¾Ĺ +è¯ķ è¯ķ +访 è°Ī +åĬł éĩį +å°± ä¸įä¼ļ +çĶŁäº§ ä¼ģä¸ļ +åĽŀ åĽ½ +åºķ 线 +赶 åĪ° +æĶ¯ éĺŁ +æĪij们 éĥ½ +éĤ® æĶ¿ +缴 èĩ³ +éĴ¢ çIJ´ +åħ ľ +çłĶ讨 ä¼ļ +æľĪ 亮 +åĿļæĮģ 以 +åħ¬å®ī éĥ¨ +éĴ¢ 管 +å°ı çĻ½ +ç½® ä¸ļ +èģ ĭ +书 åĨĻ +æĿ ı +éħį æĸ¹ +èĢĮ åıĪ +çijŀ 士 +çķĮ çļĦ +èĢģ 大 +æĪIJçĨŁ çļĦ +å¹² ä»Ģä¹Ī +ä¸ĵ项 æĸĹäºī +çŃī å¤ļ个 +èĦ± 离 +ä¸ī 个æľĪ +çłĶ究 åijĺ +æĹĭ 转 +æŀģ èĩ´ +åħį è´£ +åħįè´£ 声æĺİ +å¾Īå¤ļ çݩ家 +车 ä¸Ĭ +交 äºĴ +å·² æĺ¯ +ä¸Ģ å°ı +çļĦ éĩįçĤ¹ +èĬ± äºĨ +ä¸į æĺİ +æľīåħ³ è§Ħå®ļ +çĬ¹ å¦Ĥ +çľ ¸ +å¯ ¡ +çļĦ è¡£æľį +åĮħ 裹 +身 åŃIJ +å¸ĪèĮĥ 大åѦ +äºĭ åħĪ +线 æĿ¡ +æ³ķ åĪ¶ +åħ» æĬ¤ +稳å®ļ æĢ§ +éĤ µ +åŀĦ æĸŃ +é¡ į +èĢĥ åı¤ +æĿł æĿĨ +èĭı èģĶ +æ°´ çĶµ +åħ·ä½ĵ çļĦ +æ¿Ģ æ´» +æĪij æł¡ +åĪļ å¼Ģå§ĭ +åĩ¸ æĺ¾ +ç¦ ¾ +åħ¼ èģĮ +éĢı éģİ +åľ¨ 游æĪıä¸Ń +社ä¼ļ åıijå±ķ +好 çİ© +å¹» æĥ³ +ä¸į 代表 +注æĦı åĬĽ +æ£ į +çĶ¨ æīĭ +ç¾İ 人 +许å¤ļ 人 +å¾Ī æĺ¯ +çļĦ çłĶåıij +æīĵ åĩº +åIJĪä¼Ļ 人 +ä¸Ģ å¤ľ +ç¼ĵ ç¼ĵ +ä¿® æŃ£ +æĦŁ çŁ¥ +ç»Ī 身 +æ¿Ģ ç´ł +çݯå¢ĥ ä¸ĭ +次 ä¼ļè®® +ç»ıæµİ å¢ŀéķ¿ +æī Ľ +åıij éħµ +åĪĨæŀIJ å¸Ī +åľ¨ æľªæĿ¥ +主è¦ģ æľī +ä¸Ģ åŃ£åº¦ +çļĦ 说æ³ķ +ä»İæĿ¥ 没æľī +è´§ 车 +缩 å°ı +太 è¿ĩ +æķĪ åĬĽ +ä¸į ä¸ĭ +æĬķ 稿 +èį¯ ä¸ļ +ç»Ħ éķ¿ +ç«Ļ çĤ¹ +å¾Ī åĸľæ¬¢ +éIJ µ +åĬ¿ 头 +æ¼ı æ´ŀ +æĦ¤ æĢĴ +åħħ å®ŀ +åĪĽä¸ļ æĿ¿ +çĪ ª +æľª å¿ħ +åºķ éĥ¨ +å¾Ĺ åĪĨ +人æ°ij åĮ»éĻ¢ +äºĮæīĭ æĪ¿ +å·²ç»ı 被 +大 楼 +æĸ° æĪ¿ +辦 æ³ķ +çĶ¨ åĬĽ +æĭĵ 宽 +åĨħ åľ¨ +æĴŃ åĩº +饰 æ¼Ķ +ä¹Ł 让 +ä½ľ çĤº +çī©ä¸ļ 管çIJĨ +åį´ ä¸į +为 ä¸ŃåĽ½ +å±Ģ åĬ¿ +ä¸į èĤ¯ +æľĢ æĸ°çļĦ +åı¯ä»¥ éĢīæĭ© +æĺ¾ çİ° +å°± ç®Ĺæĺ¯ +åľ¨ æł¡ +é¾ Ł +两 æĿ¡ +çļĦ å®ŀåĬĽ +è¶Ĭ 好 +她 åľ¨ +å¿ł è¯ļ +ä¹Ł éľĢè¦ģ +游æĪı æĵįä½ľ +è¶ħ åĩº +å¦Ĥæŀľ ä¸į +æīĢåľ¨ çļĦ +ä½ł è¿ĺ +以 åĨħ +æľī ä¸Ģå®ļ +åı¯ è¾¾ +è·ij åĪ° +åī Ľ +建ç«ĭ åģ¥åħ¨ +æķ´ 车 +åīį æĸ¹ +éĹ´ æİ¥ +çŃ¹ å¤ĩ +çĸ² åĬ³ +离 å¼ĢäºĨ +æ± Ŀ +éĿ¢ éĥ¨ +ä¹ĭåīį çļĦ +åıĺ 为 +å¦Ĥæŀľ 说 +对 ä»ĺ +åĿĩ åı¯ +被åijĬ 人 +ç²¾ ç¾İ +èģļ ä¼ļ +çĿĢ æĢ¥ +è°· æŃĮ +ä¸Ģ åı· +红 åĪ© +ä¼łå¥ĩ 游æĪı +å» ĸ +è´ ŀ +ä¹° åĪ° +éŃ ļ +ä½ĵ è´¨ +å°ij äºĨ +æ³ī å·ŀ +åIJ Ł +ç»Ŀ ä¸į +é»ij æģ¶ +é»ijæģ¶ åĬ¿åĬĽ +ä¸Ĭ æĺł +çļĦè¯Ŀ é¢ĺ +ä¸ĩ人 次 +ä¸ĸ éĹ´ +çĶ¨ å·¥ +è´¯ ç©¿ +å®Ŀ çŁ³ +ä½ł 好 +åĪĩ åī² +强 åĽ½ +åĽŀ èIJ½ +æ°´ æĻ¶ +模 仿 +æ´ª æ°´ +éĢĻ 麼 +åįģä¸ī äºĶ +ä½ ij +éĻ Ħ件 +çļĦ å¢ŀéķ¿ +éĻĦ å±ŀ +çİ° å·² +帮 ä½ł +éĩij çīĮ +é«ĺ åİŁ +åľ¨ 家éĩĮ +éĺ² èħIJ +ç¡®å®ŀ æĺ¯ +宣 讲 +天 æīį +ç»ıèIJ¥ 管çIJĨ +éĶħ çĤī +åIJĪ ä¸Ģ +è§Ĥ èµı +éķ¿ è¾¾ +主ä¹ī æĢĿæĥ³ +éĤ£ 麼 +é£İ äºij +为主 çļĦ +æļij åģĩ +æĮģ ä¹ħ +å¼Ĥ åľ° +å¼Ģ éŨ +模 æĿ¿ +æī¹ 次 +ä¸į 便 +天 çĶŁ +åĩł 个æľĪ +ä¸ĵ ç§ij +åı¦ æľī +åħ¬å¸ĥ çļĦ +æĩ · +åľº åIJĪ +çļĦå¿ĥ æĢģ +è¿ĺ 好 +å®ŀ æĪĺ +èĢģå¸Ī çļĦ +åħ© åĢĭ +åı¯ åľ¨ +éĤ£ ä½į +å¥ł å®ļäºĨ +ä¿ĥ éĶĢ +æı´ åĬ© +ä¸ĩ çī© +æĥħ æĬ¥ +é¦ĸåħĪ è¦ģ +æĸĩåĮĸ åĴĮ +éĥ½ å·²ç»ı +ä¸Ĭ ä¸ĸ纪 +åĨľ åľº +大 æī¹ +æĺİçĻ½ äºĨ +çļĦ æĪIJéķ¿ +çļĦ æ¯ĶèµĽ +失 误 +åģļ æĪIJ +ä»Ĭ天 å°ıç¼ĸ +é¢Ĩ è¢ĸ +æıIJåįĩ äºĨ +å¾IJ å·ŀ +ä»į æľī +è¿ĩ 滤 +å¹½ é»ĺ +çĥŃ éĩı +ä¸Ģ é¦ĸ +æ¼Ĥ亮 çļĦ +åĩł ç§į +åĢ¡ è®® +å°±åı¯ä»¥ äºĨ +æİĴ åĪĹ +éĩį éĩį +ä¼ģä¸ļ åĴĮ +ä¸ĵ å±ŀ +çħ İ +亲 æĪļ +çĻ¾åĪĨ ä¹ĭ +稿 件 +è¿ĺ å¾Ĺ +人 åĵ¡ +äºī 夺 +æĽ´ 容æĺĵ +大 èĩªçĦ¶ +鼻 èħ¦ +太 空 +åľ° å¤Ħ +å¤ ¢ +ä»ĸ 对 +å¿ħ å°Ĩ +ä¸į å½ĵ +严 è°¨ +åĩº åľº +å·²ç»ı æľī +é¢Ĩ åĨĽ +é«ĺ æ¡£ +ä¸Ģ æīĢ +æł Ĺ +让 åѦçĶŁ +æĽ¹ æĵį +æŁIJ ä¸Ģ +伸 åĩº +èĬ± åįī +æ¸ħ éĨĴ +èģĶç³» æĸ¹å¼ı +åĪĨ å±Ģ +èħ ³ +æ©¡ èĥ¶ +éķ¿ å¾Ĺ +绿 åľ° +è¢ į +çļĦ èīºæľ¯ +女 æľĭåıĭ +ä¸Ń è¶ħ +离 åŃIJ +å¤ļæł· åĮĸ +éĺ³ åı° +ä½İ 碳 +ä¸Ģ ç±» +çŃīæĸ¹éĿ¢ çļĦ +å¾Ĺ 好 +模 åħ· +ä¸ĩ 亿 +çķĻ æĦı +临 æ²Ĥ +å°ij éĩı +çľĭ åIJij +ç»ıèIJ¥ èĢħ +çķĻä¸ĭ äºĨ +åĿı äºĨ +åijĬ åĪ« +羣 çIJĨ +ç¼´ è´¹ +æĬĬ ä½ł +çļĦ ä»»åĬ¡ +æĪij 对 +ä¹° åħ¥ +çĻ» ä¸Ĭ +æľī 两个 +ä¸Ģ 头 +æĵį æݧ +åħ¨ è¦ĨçĽĸ +çĿĢ æīĭ +å¢Ļ éĿ¢ +å¤ļ æĸ¹ +åı¯çĪ± çļĦ +ä¹Ł åı¯èĥ½ +æľĢ æľī +è¿ĻäºĽ éĥ½æĺ¯ +æĥ ¡ +å® ® +å¾Ī å°ı +éĹ®é¢ĺ æĺ¯ +åĿĩ æľī +å¾ģ éĽĨ +说 åĩº +æľī æĦı +é¢ Ĥ +æī¬ å·ŀ +åķĨä¸ļ 模å¼ı +çĶŁ èĤĸ +æįIJ 款 +å² Ĥ +ç¾İ æĻ¯ +è¿ĺ 羣 +æĭ¥ æĬ± +身ä½ĵ åģ¥åº· +æ·± å¤Ħ +çľ¼ ç¥ŀ +çļĦ 形象 +ä¼ĺ è¶Ĭ +å½ĵ æĪIJ +åĮº åĪĨ +åİ» éĻ¤ +注 å®ļ +å§IJ 妹 +åĮº åĨħ +é© ļ +æļĹ ç¤º +æĺİ äº® +æħ° éĹ® +å¸Ĥåľº 份é¢Ŀ +çĮª èĤī +çļĦ èµĦéĩij +åİĨ ç»ı +å§ĭç»Ī åĿļæĮģ +çĶŁ æľº +ä¸į 顾 +éĩij åĪļ +大 声 +éĻķ 西çľģ +é² į +åĨľä¸ļ åĨľæĿij +æľī 害 +éŨ è¯Ĭ +æ¯ı ä¸Ģ次 +çļĦ åĽłç´ł +é¢Ŀ å¤ĸ +åİ¿ 级 +çļĩ åIJİ +åĽ½ ä¼ģ +é¦ĸ éĢī +ç¼ĸ åĨĻ +æĭ¿ èµ· +åģ· åģ· +ä¸İ ä¸ŃåĽ½ +åįĸ 家 +ç»Ļ ä»ĸ们 +ç¥ŀ è¯Ŀ +åѸ æł¡ +æĪij ä¸Ģ缴 +çŁ¥éģĵ äºĨ +åį Ĵ +åĴĮ åľ°åĮº +ä»Ģä¹Ī éĥ½ +çĶ» 家 +æľ¬ çĿĢ +ä½Ļ åIJį +审 çIJĨ +ä¸Ģ åIJij +åıijå±ķ è¶ĭåĬ¿ +åĮº éĹ´ +注åĨĮ èµĦæľ¬ +çIJ ¦ +ä¸į åı¯ä»¥ +çļĦ åĦ¿åŃIJ +åĢ¼ çıŃ +ä¸¥æł¼ çļĦ +å®ŀä½ĵ ç»ıæµİ +æľī æĿĥ +æĪij åıĪ +éĵ¶ æ²³ +ç«ĭ 马 +æĿĢ äºĨ +åĮħ 容 +管 家 +身 é«Ķ +éĵ ħ +å°ı åŃIJ +管çIJĨ ç³»ç»Ł +æľīçļĦ 人 +é£İ çĶµ +æĻºèĥ½ åĪ¶éĢł +ç²¾ ç¡® +æĭĽåķĨ å¼ķ +æĭĽåķĨå¼ķ èµĦ +äºĮæīĭ 车 +åİ¿ å§Ķ +èīº 人 +å¥ ķ +è¿İ æĿ¥äºĨ +ç»ĵæĿŁ äºĨ +çļĦ ä¼łç»Ł +æĭ¼ æIJı +奥 迪 +çĸij æĥij +ä¹ĭ æĹ¥èµ· +æłĩå¿Ĺ çĿĢ +åľ° åįĢ +è¯ł éĩĬ +åĪ° æľŁ +åħ¨ éĥ½ +çŁŃ æļĤ +æĺ¯ æĪijåĽ½ +æĪij å·²ç»ı +æ»´ æ»´ +天 èµĭ +对 她 +åį«çĶŁ éĹ´ +çĶŁäº§ åŁºåľ° +æĹ¥ è®° +çļĦ æķĻåѦ +åĵ ĩ +æ°ij äºĭ +è¿ĺ åİŁ +æīĭ ä¸ŃçļĦ +çļĦ èī¯å¥½ +æ· « +ä¸Ńåħ± ä¸Ń央 +åĪ ĥ +åĵ Ħ +åľ¨ ä»ĸçļĦ +å°Ī æ¥Ń +åľº éĿ¢ +éĤ» å±ħ +çĹ Ĵ +å¦ Ħ +å¤ĸ ç§ij +ä¸į éĢĤ +举åĬŀ çļĦ +é Ĥ¹ +åħļçļĦ 建设 +çĻ¼ 表 +è·¨ çķĮ +æ²ī æ·Ģ +大 çīĩ +è¶Ĭ é«ĺ +å°Ĩ æĺ¯ +è§ī éĨĴ +åĤ¨ åŃĺ +å¢ŀ 大 +ä¸į 让 +æķ´ å½¢ +å¹³åı° ä¸Ĭ +åĩł ä½į +è¯ī æ±Ĥ +好 ä¸į好 +åľ į +æĸĩ æľ¬ +éĢ² åħ¥ +ç´ į +æł¹ æĵļ +èįī æ¡Ī +åħŃ ä¸ª +åĭ ¿ +åĪ¶ æĪIJ +饮 æ°´ +æ°¸ æģĴ +èĩª æĿĢ +åı¸ 马 +éļ¾ çĤ¹ +为 æĪij们 +å¼ § +åī© ä¸ĭçļĦ +åĩĨå¤ĩ 好 +çļĦ æľĢä½³ +èģĶåIJĪ ä¼ļ +æĤ£èĢħ çļĦ +æĪijä¸į çŁ¥éģĵ +ä¸ĭ ä¸Ģ个 +åıijå±ķ æĸ¹åIJij +ç¬ ¨ +æīĢ以 æĪij们 +åĨĻ äºĨ +éĢł æĪIJäºĨ +æ²Ļ æ¼ł +çŃĽ éĢī +çģ¾ åĮº +ä¸Ĭ çľĭ +éħ ¶ +æ»ļ åĬ¨ +éļ¾ åħį +åIJī åĪ© +ä¸Ģ ä¸Ģ +ç²¾ å¯Ĩ +伸 æīĭ +礼 仪 +åħ¨ æĺ¯ +è¶Ĭ 大 +ä¸Ń æłĩ +åıĸ åĨ³ +åıĸåĨ³ äºİ +éĢĶ ä¸Ń +讨 åİĮ +æīĭ åĨĮ +第 ä¹Ŀ +åŃĶ åŃIJ +çĦ¶ å¾Į +ä¸Ģ åħ± +æµ· æĬ¥ +款 å¼ı +æķ´ 天 +è¾¹ çķĮ +è·¯ è¾¹ +æĻĭ 级 +åIJIJ 槽 +çļĦ åħ³æ³¨ +æĪij 没æľī +å°±æĺ¯ åľ¨ +缮 çļĦæĺ¯ +åį³ä½¿ æĺ¯ +顶 å°ĸ +å·²ç»ı åľ¨ +å®īåħ¨ éļIJæĤ£ +æłĩ æĿĨ +åįĹ éĢļ +ä¼ļ 对 +座 ä½į +èµ¢å¾Ĺ äºĨ +åİŁæĿ¥ çļĦ +身 为 +书 åºĹ +è¢Ń åĩ» +ä»Ĭ æĻļ +以 èī² +以èī² åĪĹ +æĬĸ éŁ³ +åį´ 没æľī +丧 失 +çļĦ å±ĢéĿ¢ +åįģåĽĽ äºĶ +çŃī 缸åħ³ +æ±ĩ æĢ» +å¤ĸ 表 +为 æ°ij +éľĩ æĥĬ +å¥Ĺ è·¯ +çĬ¯ç½ª å«Įçĸij +å°Ĩ 以 +çİĩ é¢Ĩ +éħĴ åIJ§ +è¡Įä¸ļ åıijå±ķ +å¹´ èĩ³ +åĻ¨ æĿIJ +åĴĮ æĬĢæľ¯ +æľĢ å°ı +è¿Ļä¸Ģ åĪĩ +èģĮ 称 +å½ĵ ä½ľ +æİĢ èµ· +åĴ ĭ +ä¸Ń éĥ¨ +æīĭ èĩĤ +ç½¢ äºĨ +媳 å¦ĩ +æ´½ è°Ī +æĹ¶ä»£ ä¸ŃåĽ½ +人çĶŁ çļĦ +æŀģ éĻIJ +ç¦ Ħ +åĮº æĶ¿åºľ +æľ¬ éĴ± +礼 åĵģ +çļĦ éĤ£ä¸ª +侦 æŁ¥ +太å¤ļ çļĦ +å®ŀæĸ½ æĸ¹æ¡Ī +é«ĺ æłĩåĩĨ +æĮĩæĮ¥ éĥ¨ +åĢ¾ æĸľ +çī¹èī² 社ä¼ļ +çµIJ æŀľ +éĴ» çŁ³ +移 æ¤į +çī¹ ç§į +èĩª æĦ¿ +æĭľ çĻ» +åįķ 身 +åį´ åıĪ +åĪ¥ 人 +åIJĪ è§Ħ +æľº çĶµ +çī¹ æĦı +å½ĵåīį ä½įç½® +ä¹° 家 +åIJĪ 约 +èĤ© èĨĢ +为 åĩĨ +家 è£ħ +çļĦ çĥŃæĥħ +éĿŀ éģĹ +çļĦ éŃħåĬĽ +åİŁ åijĬ +社ä¼ļ åIJĦçķĮ +ä¹° çļĦ +å¤ļ åIJĥ +éĽķ å¡ij +èµ· ä¹ī +åĬł åī§ +éĤ£ä¸Ģ åĪ» +å°Ĩ è¿Ľä¸ĢæŃ¥ +æ¡Ĥ æŀĹ +æĽ´ 强 +对 ä¼ģä¸ļ +æĹł æĦı +ä¹łè¿ijå¹³ æĸ° +æµģ 失 +å¾® 软 +缸 对äºİ +座è°Ī ä¼ļ +主 èIJ¥ä¸ļ +主èIJ¥ä¸ļ åĬ¡ +ç§ģ åĭŁ +å±ķ示 äºĨ +常æĢģ åĮĸ +è² ´ +符 åı· +å¹´è½» çļĦ +å°± éľĢè¦ģ +ä¹Ł æĽ¾ +çļĦæĥħ 绪 +è¾¾ æłĩ +èĩ ¨ +ä½į å±ħ +ä»ħ 为 +é¦ĸ 家 +éĺ´ éĺ³ +ä¸įåĨį æĺ¯ +åĽłä¸º å®ĥ +ä¼ģä¸ļ åľ¨ +çĺ ¾ +åIJ¬ è§ģ +åİŁ æľī +åĪ¶ è£ģ +å¯Ĥ å¯ŀ +éĢļè¿ĩ 对 +æ»ij éĽª +è¿Ļ å¼ł +çļĦ çIJĨ解 +æĸ° ä¸ŃåĽ½ +è¿Ļ åĦ¿ +ä½İ ä»· +æĥ³ è¿ĩ +çļĦ ä¿¡å¿ĥ +建çŃij çī© +çļĦ é¢ľèī² +ä¸į åºĶ该 +æĹłçĸij æĺ¯ +å¼ķèµ· äºĨ +åħ¨ åijĺ +æĿ° åĩº +è¿Ļæĺ¯ æĪij +èª ° +èĺ ĩ +éĺµ åľ° +åħħ åĢ¼ +çŁ¿ ä¸ļ +çĿĢ ä»ĸ +ä¿¡ 访 +ä¸ĩ è¾¾ +æij© æĵ¦ +å¼Ģ 端 +èı² å¾ĭ +èı²å¾ĭ 宾 +车 åŃIJ +æľ¬èº« çļĦ +çģ«è½¦ ç«Ļ +常 å·ŀ +为 代表 +为代表 çļĦ +广 çĶµ +亲 人 +åı³ æīĭ +éĽĨ è£ħ +éĽĨè£ħ ç®± +çļĦ åį°è±¡ +æ©Ł æľĥ +åĮĨ åĮĨ +åħī çĶµ +大 æĸ¹ +è¿ĺ æľª +åĪ© 好 +ç»Ŀ 大å¤ļæķ° +åľ¨ è¿Ļç§į +ä¸Ģ ç»Ħ +æĸ° èĤ¡ +转 åıij +æ³ķ åºŃ +æĹł æīĢ +éģĵ è·¯ä¸Ĭ +çŁ¿ å±± +èij ī +æĶ¶ åĽŀ +称 ä¹ĭ +称ä¹ĭ 为 +æıŃ éľ² +åı£ 岸 +åIJ ¼ +å¿ĥ æĥ³ +çļĦ 梦æĥ³ +éĽ ¯ +ä¹ĭ åĪĿ +å¥ĸ 项 +订 éĺħ +èĵĿ 天 +åĿ¦ åħĭ +ç«ĭ æ¡Ī +èģĶ æīĭ +ä½Ĩæĺ¯ æĪij +帮 æĪij +ä»ħ 代表 +说 æĪij +çļĦ è¶ĭåĬ¿ +æ¯Ķè¾ĥ 大 +èµ° å»Ĭ +éĩįçĤ¹ é¡¹çĽ® +èµĮ åľº +åIJį çīĩ +æĦŁ åı¹ +åľ¨ åľ°ä¸Ĭ +åıij çĥŃ +èĮĥ çķ´ +çļĦ éģĵè·¯ +éĩij èī² +ä»ĸ åıĪ +ä¼ļ 产çĶŁ +æ°ij åĽ½ +å®ĺæĸ¹ ç½ijç«Ļ +æĶ¶çĽĬ çİĩ +çļĦ åĪ°æĿ¥ +çļĦ åĬŀæ³ķ +æĶ¹ åĪ¶ +ä¸ĩ ç§ij +ä¸į äºĪ +è¿ĻäºĽ éĹ®é¢ĺ +çĪ± ä¸Ĭ +çIJĥ åľº +è´£ 令 +æİĪ 课 +åľ¨ é¦Ļ港 +ç»Ĩ èħ» +å¤ļ ä¸ĩ +åIJĮ å¹´ +大 使 +æĸ ĭ +ä¹Ł 为 +æĥł å·ŀ +åIJī 祥 +çĶ° åĽŃ +åĽ½å®¶ éĺŁ +éĩį çĶŁ +åľ¨ åħ¶ +é¦Ļ åij³ +è´Ł èį· +亲 åĪĩ +èĩª 豪 +没 éĶĻ +åĽłä¸º åľ¨ +æĺŁ æĺŁ +éĤ ij +è¿ĺæľī å¾Īå¤ļ +æij© æīĺ +æij©æīĺ 车 +æŃ¥ è¡Į +管çIJĨ ä½ĵç³» +èĦļ ä¸ĭ +éģİ åİ» +æ±ī è¯Ń +对 ä¸įèµ· +çļĦ ç»ıåİĨ +åıĬ 缸åħ³ +ä¸įå°ij 人 +éĩį ç£ħ +åĬ³åĬ¨ èĢħ +大åĬĽ åıijå±ķ +æĢİä¹Ī åģļ +çĭĹ çĭĹ +举åįĹ äºļ +åĭĩ äºİ +åħ¬ éĸĭ +çĵ· çłĸ +åıĤ çħ§ +广æĴŃ çĶµè§Ĩ +举 åĬ¨ +æ±Ł 西çľģ +æķĪ èĥ½ +åĶ¯ æľī +éĿ¢ è²Į +èĩªåĬ¨ 驾驶 +æ¦ľ åįķ +å½ĵ æĪij们 +仲 è£ģ +æľ¨ æĿIJ +ç±³ åħ° +çĻ½ éĵ¶ +çļĦ 人éĥ½ +å°± åĥıæĺ¯ +æŃ¥ åħ¥ +åįł çĶ¨ +åĩ» è´¥ +让 大家 +ä¼ļ è®©ä½ł +åİ¿ æĶ¿åºľ +è¦ģ çĶ¨ +çŃī å½¢å¼ı +åįĩ é«ĺ +责任 æĦŁ +å¤ĩ çĶ¨ +ä»ĸ 认为 +æ¸ħåįİ å¤§åѦ +ä»ĸ èĩªå·± +éĸ± è®Ģ +太平 æ´ĭ +éĶģ å®ļ +çŃ Ĩ +è¿Ļ çīĩ +æī§ æĶ¿ +è¿ĶåĽŀ æIJľçĭIJ +å°± æѤ +éģĩ åĪ°äºĨ +å¼Ģå¹ķ å¼ı +管çIJĨ éĥ¨éŨ +姿 åĬ¿ +设 æĥ³ +åĽĽ åŃ£ +æĬĢæľ¯ 人åijĺ +å·® çĤ¹ +è¾ŀ èģĮ +èĢģ 師 +çļĦ æĦŁåıĹ +ä¹Ł éĿŀ常 +å¹´ ä¸ĬåįĬå¹´ +æĢª çī© +èĮĥ æĸĩ +æĪĺ å½¹ +åIJ« ä¹ī +åħ¨ è¿ĩç¨ĭ +èĢĮ éĿŀ +éĢļ讯 åijĺ +è¿Ļæł· æīįèĥ½ +æľº ç»Ħ +è£ ı +çķ¶ çĦ¶ +èµĮ åįļ +åIJĦ æľī +å·¥ä½ľ æľºåĪ¶ +äºĭ åIJİ +åī§ éĻ¢ +å±Ĭ æŶ +åĺ´ éĩĮ +主 线 +ä¸Ģ åľĪ +主è¦ģ åİŁåĽł +å°¸ ä½ĵ +åĮ»çĸĹ åĻ¨æ¢° +ä½ł æĢİä¹Ī +ä½Ĩ çĶ±äºİ +æŶ 空 +çĶ· æľĭåıĭ +çĶľ èľľ +é«ĺ åľ° +æĻ ĸ +èĴIJ éĽĨ +åĩĿèģļ åĬĽ +å¤ĩ åıĹ +æĸĩ åĪĽ +马 æĿ¥ +马æĿ¥ 西äºļ +æŁ´ æ²¹ +使 人 +æķĻ ä¼ļ +ç§ĭ 天 +æĺİ çıł +åħŃ åįģ +çݯå¢ĥ ä¸Ń +æ¸ħ æĻ¨ +积æŀģ åıĤä¸İ +å·ħ å³° +为 æľŁ +çŃ¾ åŃĹ +æĦŁ æ¿Ģ +ç§ĭ åŃ£ +æĿij åŃIJ +æ¢ħ 西 +æļ´ 鼨 +çĶŁæ´» åľ¨ +çªĹ æĪ· +æģ¶ åĬ£ +纯 ç²¹ +åľ¨ æİ¥åıĹ +没 èĥ½ +è¡Į 人 +åĭ º +æĭ¨ æīĵ +ä½ľ åĩºäºĨ +çļĦ 主é¢ĺ +æľª ä¾Ĩ +ä¸Ń æľĢ +æ¾ ľ +é«ĺ è¡Ģåİĭ +åħ´ èµ· +æŃ£ èĥ½éĩı +åŁ¹è®Ń çıŃ +æİ¥ åħ¥ +çĦ¶åIJİ åĨį +åѦçĶŁ 们 +é¢ĨåħĪ çļĦ +çģ« çĥŃ +ä¸ĵ èģĮ +æĪĸèĢħ 说 +建 è¨Ń +é» ı +对 åħ¬åı¸ +çī¹ æľīçļĦ +åħī èį£ +å½ĵ åľº +éĿ¢ åŃIJ +èµĦ产 管çIJĨ +æĹ¶æľŁ çļĦ +çŀ İ +åįİ ä¸ľ +åıĪ ä¸Ģ次 +èĥİ åĦ¿ +å®ļ çĤ¹ +头 çĹĽ +液 ä½ĵ +æĺ¯ä¸Ģ ä½į +帽 åŃIJ +å¹´ èµ· +ä¸į ä½İäºİ +è¾ĥ å°ij +éĿ¢ä¸´ çĿĢ +å±Ĥ å±Ĥ +èĿ´ èĿ¶ +èī° èĭ¦ +éĺ¿ æł¹ +éĺ¿æł¹ å»· +æ¦Ĥ æĭ¬ +请 éĹ® +èµ· åºĬ +å±Ģ å±Ģéķ¿ +稳 åģ¥ +å¦Ĥæŀľ æĪij们 +éħĴ ç²¾ +æĪ· åı£ +æĦŁ æĤŁ +æĪij们 éľĢè¦ģ +æĬĢ èīº +èĩª åªĴä½ĵ +è¿Ľ åĮĸ +æ¿ĢçĥĪ çļĦ +ä½ĵ 温 +èļ ķ +èĩ´ è¾ŀ +宪 æ³ķ +ä¸Ģ çŃīå¥ĸ +çĵ¶ é¢Ī +æĥł æ°ij +èµ° è·¯ +çİ° ä»» +åķĨ éĩı +ä¸ĭ 车 +åĪ ł +責 ä»» +èŀįåIJĪ åıijå±ķ +ç´ł æĿIJ +æ²¹ ä»· +åģļ 人 +çŀ ª +æĶ¹éĿ© åĪĽæĸ° +çļĦ åĮºåĪ« +è·¨å¢ĥ çĶµåķĨ +æ¶īåıĬ åĪ° +æīĺ 管 +æĪij è¿ĺæĺ¯ +åĿIJ æłĩ +ç½ij 讯 +å½ĵåľ° çļĦ +追 溯 +åľŁ èĢ³ +åľŁèĢ³ åħ¶ +åºķ ä¸ĭ +åĩł åįģå¹´ +ç©¿ è¿ĩ +çĶŁæĢģ æĸĩæĺİ +æݨ èĸ +æݨèĸ ¦ +éł Ĩ +åĴ³ åĹ½ +åĪĨ æĪIJ +çĹķ 迹 +æĪ· ç±į +éĥ½ ä¸įèĥ½ +æĻļ ä¼ļ +åĢ © +ä½ĵ åĬĽ +è¿Ļ个 èģĮä¸ļ +æĹł å½¢ +åıª æĥ³ +è¿Ľ åıĸ +æĿĢ æŃ» +èĦ Ĭ +äºij åįĹçľģ +æľª çŁ¥ +ç¾İ èģĶ +ç¾İèģĶ åĤ¨ +å¤ĸ å½¢ +诱 æĥij +çĽ £ +è¡Į 使 +åłĨ 积 +çĨŁ ç»ĥ +éĺIJ è¿° +æľĢ大 éĻIJ度 +å·¡ æŁ¥ +夺 åĨł +ä¼ģä¸ļ æĸĩåĮĸ +çĭ® åŃIJ +ä¿Ŀ å®Ī +ä¸ºæł¸å¿ĥ çļĦ +æī© æķ£ +åĪ¶éĢł åķĨ +æŁĶ 软 +为ä¸Ģä½ĵ çļĦ +游 çİ© +çĶŁ çĹħ +幫 åĬ© +åĶ± æŃĮ +æīį åı¯ä»¥ +宽 æĿ¾ +è¦ģ æ¯Ķ +æĺ¯ æĢİæł· +çģ° èī² +çİĭ åĽ½ +æIJħ æĭĮ +计 éĩı +åij¨åĽ´ çļĦ +æĻºèĥ½ æīĭæľº +常 åĬ¡ +常åĬ¡ åī¯ +é© ´ +å°Ĩ è¿ij +寻 常 +ä¸ŃåĽ½ å¸Ĥåľº +容 åĻ¨ +å±± ä¸Ĭ +èĥĮåIJİ çļĦ +亲 å¯Ĩ +æīĢ以 说 +éİ ® +çļĦ çIJĨçĶ± +大 åŁİå¸Ĥ +常 å¹´ +æĹħ游 ä¸ļ +å°±æĺ¯ è¿Ļæł· +åĨį æĿ¥ +é«ĺ ä½į +åĨħ 饰 +æŀĦ éĢł +ä¸Ģ èµ·æĿ¥ +çĶ³ è«ĭ +å·²ç»ı å¼Ģå§ĭ +çļĦ åĬ¨ä½ľ +被 è¿« +éģį å¸ĥ +åīĸ æŀIJ +å°ı äºĭ +å¿ĥ ä¸ŃçļĦ +ä½ĵåĪ¶ æĶ¹éĿ© +çļĩ 家 +æķĻ åłĤ +åIJĥ å®Į +åĽ½æ°ij åħļ +æĺİç¡® äºĨ +åıijå±ķ è§ĦåĪĴ +第ä¸Ģ æŃ¥ +å¾Ĺ èµ· +åľ¨ åĵª +çļĦ è·¯ä¸Ĭ +é» Ķ +çķ¶ æĻĤ +大åĬĽ æĶ¯æĮģ +åıĮ éĩį +çŁ¥éģĵ èĩªå·± +åIJĪä½ľ åįıè®® +æ°Ķ åĬ¿ +éķ¿æķĪ æľºåĪ¶ +ç½ķ è§ģ +åĽŀ æĿ¥äºĨ +ä»ĸ ä¼ļ +ä¸Ń æĸ° +ä¸Ńæĸ° ç½ij +çļĦ åķĨåĵģ +èµł éĢģ +決 å®ļ +å¸Ĥåľº çĽij管 +çķĻ åѦçĶŁ +çĶµ åİĭ +äºļ 马 +äºļ马 éĢĬ +è¿ĺæĺ¯ æ¯Ķè¾ĥ +ä¿ĥè¿Ľ äºĨ +æµģ åħ¥ +æijĦ åĥı +æijĦåĥı 头 +æıIJ åıĬ +åıij æİĺ +æī¾ åĩº +æ¢Ŀ 件 +ç¹¼ çºĮ +æĪij åĸľæ¬¢ +å¥ İ +æ¦ľ æł· +å¼Ģ èĬ± +æ²ī éĩį +åŁº åĩĨ +ä»ħä»ħ æĺ¯ +轨éģĵ 交éĢļ +åĶIJ å±± +çŃī ä¸Ģç³»åĪĹ +ä¸įè¿ĩ æĺ¯ +åŃĺåľ¨ çĿĢ +èĬ± çĶŁ +å¤ · +ç»Ī 究 +ä¹Łæĺ¯ ä¸Ģ个 +åįģ åŃĹ +èĸª éħ¬ +伤 å¿ĥ +æĺ¥ ç§ĭ +åĨ· åį´ +ç²¾ çģµ +çļĦ åľ°åĽ¾ +æ¯Ķ çī¹ +æ¯Ķçī¹ å¸ģ +æĢ§ åĪ« +ä½Ļ ä¸ĩåħĥ +ä¸įå¿ĺ åĪĿå¿ĥ +å¿ĥ çĸ¼ +æĽ² 线 +é«ĺ ä½İ +è¦ı å®ļ +æĻ¯ èī² +è¦ģ 说 +åħ¬åı¸ å°Ĩ +液 åİĭ +è¿Ŀ 约 +åİļ 度 +åºŀ 大çļĦ +è¿ĺæĺ¯ å¾Ī +é¦ĸåħĪ æĺ¯ +çµ ² +åĬ¡ å®ŀ +並 ä¸Ķ +å¢ŀ è¿Ľ +ç»Ħç»ĩ å¼Ģå±ķ +èµ·æĿ¥ äºĨ +è¾ĥ å°ı +导 游 +两 åľ° +ç¿ ĺ +çģ¿ çĥĤ +é£İ éĩĩ +æĶ¯ 线 +æĶ¯çº¿ ä»»åĬ¡ +娱ä¹IJ åľĪ +天津 å¸Ĥ +åĮħ åĽ´ +æľ¬ èµĽåŃ£ +éĩįè¦ģ 讲è¯Ŀ +åıĮ åIJij +åįİ ä¸½ +éĶ ¤ +åĦ¿ 女 +åįĸ åĩº +ä¾Ĩ 說 +ä»ĭç»į ä¸Ģä¸ĭ +åIJ¦ 认 +åĭ Ŀ +æĻ®éĢļ 人 +çļĦ åĬ¨åĬĽ +涨 åģľ +åŁºéĩij 管çIJĨ +ä¸Ģ个 éĩįè¦ģ +è¿IJ æ²³ +çħ ŀ +è´¢æĶ¿ éĥ¨ +è¡Įä¸ļ åįıä¼ļ +éĥ½ å°Ĩ +è¨Ģ 论 +ä¸ĭ ä¾Ĩ +墨 西 +墨西 åĵ¥ +åĽłä¸º ä»ĸ们 +æĢİä¹Ī åĽŀäºĭ +åĬłå¤§ 对 +èĬ Ń +çīĮ åŃIJ +ä¼ļ 使 +妹 åŃIJ +ç«Ļ éķ¿ +å¿ħ å¤ĩ +æłij æľ¨ +æģ¶ æĦı +æ²³ éģĵ +å¯Į è£ķ +ç¹ģ åįİ +代表 åĽ¢ +æµij 身 +é¦ĸ ä½į +èĪªç©º åħ¬åı¸ +鼻 å½± +ä¸ĵ è¾ij +æ°´ æºIJ +ä¸Ń æ¯Ĵ +並 ä¸į +èĢĮ åİ» +é ĥĿ +äºİ æѤ +æĸĩåĮĸ 建设 +èĤ¯å®ļ ä¼ļ +å¸ĮæľĽ 大家 +æıı åĨĻ +ä½İ è°ĥ +æĸ°åħ´ 产ä¸ļ +æ·Ħ åįļ +æĶ¾ å¼Ģ +çļĦ æĢ§æł¼ +çĸ¾çĹħ çļĦ +æķ´ é¡¿ +线ä¸Ĭ 线ä¸ĭ +éĢī 项 +çļĦ 认åı¯ +æķ´ é½IJ +çĶļ ä¹Ī +çľģ åĨħ +åı¤ 人 +æ°ij ä¿Ĺ +çī¡ 丹 +éŨ çªĹ +éĤ£ æł·çļĦ +çĽijäºĭ ä¼ļ +ç¿¡ ç¿ł +ç¦ ¹ +åįĥä¸ĩ ä¸įè¦ģ +æĶ¶ 缩 +çļĦ æĸĩåŃĹ +åĴĮ å°ļ +æĮĩ 令 +åħ±äº§ åħļåijĺ +çļĦ çĪ¶äº² +å®Į å·¥ +åĬ¡ å·¥ +马 æĭī +马æĭī æĿ¾ +æµĭ è¯Ħ +å² ļ +ä¸į åģļ +ä¸ĥ å¹´ +åĿĩ ä»· +主 è§Ĥ +å¾Ī ä¸įéĶĻ +èĤ¡ä¸ľ 大ä¼ļ +äºĶ ä¸Ģ +é£İ åIJ¹ +å¼Ģ éĩĩ +è¿Ļä¹Ī 大 +èĥ½ çľĭåĪ° +èĢĥ è¯Ħ +åį³ 便æĺ¯ +çݰ代 åĨľä¸ļ +æ¯Ķè¾ĥ é«ĺ +è¦ģ çľĭ +没 äºĨ +解 決 +çݯ æ¯Ķ +åĨ² åĬ¨ +æ·± å¤ľ +åĩł åįĥ +ä¿ ı +ç½ij æ°ij +å°± 没 +ä»ĸ 表示 +éĩı åŃIJ +æĹ©é¤IJ åĬłçĽŁ +åįĬ å²Ľ +æIJŀ ç¬ij +ä¸Ĭ æĬ¥ +å¯ © +é¢Ħ 订 +èľĤ èľľ +æŁ¥ æī¾ +ä¼Ĺ æīĢ +ä¼ĹæīĢ åij¨ +ä¼ĹæīĢåij¨ çŁ¥ +æĹ© æĹ¥ +åıij æī¬ +åĴĮ 个人 +åĬłåħ¥ äºĨ +åĸ® ä½į +åĪĨ æĺİ +第ä¸Ģ æī¹ +ç¾İ åĨĽ +æĿĢ æīĭ +éŨ å¤ĸ +åķĨ åľĪ +ä¸Ģ åĪ» +çļĦçľ¼ ç¥ŀ +éľ Ħ +äºĽ ä»Ģä¹Ī +åĬł æ·± +æ¯ı ä½į +å¸Ĥ éĿ¢ä¸Ĭ +åıĶ åıĶ +çļĦ éĤ£ç§į +粤 港澳 +è´´ å¿ĥ +æĸĩåĮĸ 产ä¸ļ +红 æĹĹ +åĺī åħ´ +æĶ¶ çĽĺ +å®ĮæĪIJ åIJİ +ä¼ģä¸ļ 管çIJĨ +纵 横 +ä¸į ä¿¡ +æĪIJ éĥ½å¸Ĥ +æ´Ĺ 澡 +举è¡Į çļĦ +çĶ¢ çĶŁ +ç©¿ ä¸Ĭ +åĪļ 好 +åħī 线 +æīĵ æŀ¶ +è¿Ļ æľ¬ä¹¦ +åĶ®åIJİ æľįåĬ¡ +åĩł åĪĨ +ä¸Ĭ 次 +ä¸į åĪĨ +产 åIJİ +éģ¿ å¼Ģ +ç»Ī æŀģ +代表 大ä¼ļ +æ¼Ķ æĬĢ +åĽŀ è´Ń +åѦ è´¹ +éĺ» ç¢į +ä¸Ģ大 æī¹ +ç«£ å·¥ +åĨ³ å®ļäºĨ +ä½Ĩ å¦Ĥæŀľ +çĶµ æµģ +ä¸Ŀ 毫 +èĥ½å¤Ł åľ¨ +éĶĢåĶ® æĶ¶åħ¥ +åľ¨ åŃ¦æł¡ +æ°´ åĩĨ +è§Ĩ 线 +èĩª åľ¨ +åķĨä¸ļ éĵ¶è¡Į +为äºĨ 让 +çį² å¾Ĺ +çݩ家 æľĭåıĭ +éĿ¢ èĨľ +åĪĨ åī² +åī§ æľ¬ +ç« Ń +说 å¾Ĺ +æĥ³ çŁ¥éģĵ +çļĦ人 çī© +èĮħ åı° +åIJĮ ä¸Ģ个 +æķ°æį® ä¸Ńå¿ĥ +çĶ Ħ +åĸľ æĤ¦ +ä¸ĭæĿ¥ çļĦ +å®ļ åIJij +æŀģ åħ· +çļĦ åľŁåľ° +éĤ£ åĢĭ +æijĦ åħ¥ +äºĨ æĪijçļĦ +马 è·¯ +åħ¨ 社ä¼ļ +è®® æ¡Ī +å±ĭ åŃIJ +åIJį åı« +åĮ ª +åľ¨ å¤ĸéĿ¢ +åįİ åįĹ +åıij è´§ +å¯Ĵ åĨ· +é«ĺçŃī æķĻèĤ² +详ç»Ĩ çļĦ +个 é¡¹çĽ® +çĶŁäº§ åĬĽ +æŶ 常 +å°± æľĥ +ä¸ĩ èĤ¡ +éĻĮçĶŁ 人 +æıı ç»ĺ +å½ĵ çĦ¶æĺ¯ +æĭī åĬ¨ +éĵ¾ æĿ¡ +æī£ éĻ¤ +ä¸Ģ缴 éĥ½ +å°ı åŃ©åŃIJ +伤 åı£ +第äºĮ å±Ĭ +è´Ń ç½® +çļĩ 马 +æĹł èģĬ +表 åĨ³ +诸 å¦Ĥ +åĵį èµ· +é£İ æļ´ +ä¸Ģæµģ çļĦ +ç ·¨ +解æĶ¾ åĨĽ +室 å¤ĸ +å°± è¿Ļä¹Ī +å³ ¶ +æīĢæľī 人éĥ½ +æIJľç´¢ å¼ķæĵİ +çļĦ æĪIJæľ¬ +åħļ æĶ¿ +åıijè¡Į 人 +çļĦ äºĭå®ŀ +对 该 +åıĹ æįŁ +ä¿Ħ ä¹Į +é²ľ èĬ± +åĨľ èį¯ +æŀģ éĢŁ +æĢ¥ æĢ§ +两 ä¼ļ +ä¸ĢèĪ¬ æĿ¥è¯´ +æµ· é²ľ +åĨ Ī +çĶ¨ 人 +çĶ¨äºº åįķä½į +åĢ ª +åĦª æĥł +æł¹ æºIJ +åĽ¢ è´Ń +ç¾İ æ´² +ä¸ĭ è¡Į +å¹´ æľ« +èľ ¡ +è¯ģ 件 +åľ¨ æĪijåĽ½ +ä¸į åºĶ +æĮī æŶ +åłª 称 +åľº ä¸Ĭ +å¹²éĥ¨ èģĮå·¥ +æľī å¾Ī大çļĦ +æķ°åŃĹ ç»ıæµİ +æ¼Ķ ç»ĥ +æį® ç»Łè®¡ +å¾Ģ æĿ¥ +广åijĬ æľįåĬ¡ +çļĦ è·Ŀ离 +æŃ ¸ +è¨Ģ è¯Ń +被 èªī +被èªī 为 +åĭī 强 +å°Ĭ æķ¬ +ä¸ĩ 亿åħĥ +ä¸ŃåĽ½ åĽ½éĻħ +å¹² é¢Ħ +å¹´ 产 +èĢķ åľ° +èĮ İ +åį³ æĺ¯ +æĺ¨ æĻļ +æĪIJ为 ä¸Ģ个 +çºł æŃ£ +åij½ åIJį +é¢ģ å¸ĥ +çĮľ æµĭ +ä¿ĿèŃ· æĶ¿çŃĸ +æĭ ¢ +æ´» æ³¼ +çŃī éĥ¨éŨ +åѦ åĪ° +å¢ŀåĢ¼ ç¨İ +èĪª 线 +åĨ ¤ +åįģ åĩłå¹´ +æݧèĤ¡ èĤ¡ä¸ľ +ä¸Ģ éŨ +个 å·¥ä½ľ +ä¸ªå·¥ä½ľ æĹ¥ +æĸ° 西 +æĸ°è¥¿ åħ° +论 è¯ģ +ä» Ĩ +åı¦å¤ĸ ä¸Ģ个 +æĶ¹ ç¼ĸ +严 ç¦ģ +åĸľ 好 +个人 ä¿¡æģ¯ +满æĦı 度 +åĵ ¨ +å¸Ī èµĦ +æĶ¹ 为 +ç«ŀäºī 对æīĭ +åĩº çĤī +åķĨ 人 +大 æ£ļ +æĮĩ导 ä¸ĭ +å¦ĩ ç§ij +è¼ ª +æī ģ +åIJĮæŶ è¿ĺ +并 éĢļè¿ĩ +æĪĺ éĺŁ +èĶĵ 延 +ä¿ ŀ +éĢĤå½ĵ çļĦ +åīį è¾Ī +åĵģ åij³ +湿 åľ° +æĪIJ åŀĭ +ä¸į åıªæĺ¯ +æĥ© ç½ļ +åĩºåı° äºĨ +çİ© 游æĪı +æīį åıijçİ° +åºĶ èģĺ +å¤ĸ æĿ¥ +åįł é¢Ĩ +å±ķ æľĽ +å« Ĥ +港 èĤ¡ +æ¡Į ä¸Ĭ +æĶ¯ æŁ± +çļĦæĥħ å½¢ +广éĺĶ çļĦ +æĶ¯ è¡Į +å´© æºĥ +æľĪ ä¸Ń +æľĪä¸Ń æŬ +ç»į åħ´ +临 è¿ij +æĬ¤ æłı +æļ ® +åįķ èģĮä¸ļ +è¾¹ å¢ĥ +æĹ¥ çħ§ +ä¸Ģ åłĨ +缴 å¾Ħ +åħ±åIJĮ ä½ĵ +æĸ°åįİ ç½ij +æīĵ 好 +çĶµåĬ¨ 汽车 +ä¸į æĺİçĻ½ +éĢĻ 裡 +缼 大 +çİĭ æľĿ +åĨį ä¸Ģ次 +åĬŀåħ¬ åİħ +è´¨ æĬ¼ +åIJĪ åĩ» +人们 对 +鼶 é£Ł +éĥ½ä¸į çŁ¥éģĵ +çļĦ è¯Ńè¨Ģ +åĭŁéĽĨ èµĦéĩij +åĬ¨ èĦī +å½ ¤ +è¿Ļ åĩłå¹´ +çŁŃ è§Ĩé¢ij +太 é«ĺ +常 å§Ķä¼ļ +åĬł çıŃ +éĩį å¿ĥ +åªĴä½ĵ æĬ¥éģĵ +没 æ³ķ +éĹ» åIJį +çĥŃ åº¦ +å¹¿æ³Ľ çļĦ +åħŃ å¤§ +çī© ä½ĵ +ä¸į 该 +é¢ĺ 主 +精彩 çļĦ +为 è¿Ľä¸ĢæŃ¥ +èĻ ŀ +åĽº çĦ¶ +è´µå·ŀ çľģ +çºł ç»ĵ +代çIJĨ 人 +æ³ķå®ļ 代表 +åı¦ä¸Ģ ç§į +ä¸į åIJ« +æĭ¯ æķij +ä¼ļ ç»Ļ +è¯Ĺ è¯į +åIJĮ ç±» +å¾Ĺ ä¸įåĪ° +æĬĵ ç´§ +以 åħ¶ +åħ¥ åħļ +è¿ĺ åı¯ +æľŁ åĪĬ +å¾Īå¤ļ æŶåĢĻ +æĹ¥ åIJİ +åħ¬ 约 +ä¸Ģ 举 +æ¯Ķè¾ĥ å¤ļ +éĩij æ²Ļ +æį ŀ +æİĴ åĩº +æѦ æľ¯ +ä¸į æĸ· +ä¸Ń èĢĥ +ä¿¡ èµĸ +ä»İä¸ļ 人åijĺ +çģ« çĦ° +éĨĴ æĿ¥ +ä½İ 温 +éĢ¾ æľŁ +åĬ± å¿Ĺ +éħ ¥ +åı¯è°ĵ æĺ¯ +è¿Ļ æĦıåij³çĿĢ +é¢ł è¦Ĩ +åĮĹ京 大åѦ +ä¸ĵ 线 +åıĬ 以ä¸Ĭ +è¨ ª +èĢĮ åIJİ +çŁ¥ ä¹İ +ä¸Ģ对 ä¸Ģ +å¨ĥ å¨ĥ +çģ¾ éļ¾ +åħ¨ å±Ģ +æīĢå¾Ĺ ç¨İ +å®ŀ æĥł +èļĤ èļģ +ä¹Ł çŁ¥éģĵ +温 åĴĮ +èIJ½ ä¸ĭ +åŀĭ ä¼ģä¸ļ +åĨį ä¹Ł +ä¾Ľ çĥŃ +é«ĺ æ½® +çĢı覽 åĻ¨ +çļĦ 巨大 +åħĪ 天 +å¹´ ä¸ŃåĽ½ +类似 çļĦ +çIJĨäºĭ ä¼ļ +空 éĸĵ +çģµ æĦŁ +åĬĽ æ°Ķ +带 ä¸Ĭ +ä¸į好 æĦıæĢĿ +æľī ä½ķ +å·² åľ¨ +åıĸ åĩº +è¿Ŀæ³ķ çĬ¯ç½ª +åŃ¦ä¹ł 贯彻 +åľ° 带 +楼 梯 +çŃī æĥħåĨµ +ä»İ åīį +çļĦ ä¹łæĥ¯ +ç³Ł ç³ķ +å°± èĥ½å¤Ł +è© ķ +ä¸Ģ å¾ĭ +æĮ« æĬĺ +åİŁæĸĩ åľ°åĿĢ +å½ĵ å±Ģ +ä¸į éĢļ +æķ° åįĥ +éĺŁä¼į 建设 +æŶ èĬĤ +åģļ èµ· +çļĦ è®°å¿Ĩ +ç½ij绾 å®īåħ¨ +åĩ¡ æĺ¯ +æ° ¯ +éĽķ åĪ» +åŁĥ åıĬ +æĪij åı¯ä»¥ +çĽij çIJĨ +æĽ´ åħ· +åŁİ 管 +èĭ ¯ +åı¥ åŃIJ +èĭ¥ æľī +ä»İæĿ¥ ä¸į +缸åħ³ è´Łè´£ +å®īåħ¨ æĦŁ +æĽ´ è¦ģ +çļĦæĥħ æĦŁ +çī¢ çī¢ +è¾ĥ 好çļĦ +æ° ® +ç¬ij è¯Ŀ +车 å±ķ +ä¹ĭ ç¾İ +ç®Ģ 约 +ç±»åŀĭ çļĦ +èĢģ åĮĸ +çľĭ ä½ł +è¿ĩ åĪĨ +éŨ åīį +ä¸Ģ éĹ´ +æĥ³ åİ» +åª Ľ +åľŁ è±Ĩ +åıĪ 称 +ä¸Ń ä¿¡ +åŃĺ éĩı +马 äºij +èĩ´ 使 +åħĪ åīį +èĢģ åŃIJ +æīĵ æī® +æ¯ķä¸ļ äºİ +æ¯ķä¸ļ åIJİ +ç¾İ好 çĶŁæ´» +å·¥ä¸ļ ä¼ģä¸ļ +就好 äºĨ +èħIJ èļĢ +çıį çıł +åĪ° è¿ĻéĩĮ +æīĢéľĢ çļĦ +è¿Ļæĺ¯ åĽłä¸º +çIJĨæĥ³ çļĦ +å·®å¼Ĥ åĮĸ +é ® +é® ® +äºļ 太 +æĹł ç©· +æıIJ çİ° +ä¸ĵä¸ļ æĬĢæľ¯ +çĶ¢ æ¥Ń +åѦ åŃIJ +ç§ij å¹» +åįłåľ° éĿ¢ç§¯ +ä¸į åĩĨ +æľªæĪIJ 年人 +æĶ¶ å½ķ +è¿ĺ 款 +éĴ¢ çŃĭ +æ¼ ¢ +å¾Ĺ æĦı +综åIJĪ ä½ĵ +æŀģ é«ĺ +åįķ è¯į +é«ĺæķĪ çļĦ +骨 头 +æī§ çĿĢ +缼 ä¸ĸ +模 çī¹ +æĽ´ èĥ½ +ç»Ŀ æľĽ +对åºĶ çļĦ +æ¨ Ĭ +æĸ° ä¸ī +æĸ°ä¸ī æĿ¿ +æģ° æģ° +åIJį 家 +æł¸å¿ĥ æĬĢæľ¯ +个 å°ı +æĢİä¹Ī ä¼ļ +说 ä¸įå®ļ +西 çĵľ +åĵ İ +ç¢ Ł +å¿ħ ä¸įåı¯ +å¿ħä¸įåı¯ å°ij +ä¹ĭ éĸĵ +åĪĨ 管 +交éĢļ äºĭæķħ +å¼Ģ åĬŀ +å¾ģæ±Ĥ æĦıè§ģ +äº ¨ +鼻åŃIJ éĥµ +鼻åŃIJéĥµ 件 +ä¿¡æģ¯ æľįåĬ¡ +ä½ł è§īå¾Ĺ +缴 è§Ĥ +å·² å®ĮæĪIJ +åĪĨ ä¼ļ +åĽŀ åįĩ +éļ » +好 人 +äºĨ解 ä¸Ģä¸ĭ +åį« æµ´ +æľĢ çĪ± +åºŀ 大 +客 æĪ¿ +çijŀ åħ¸ +éĥ½ ä¸įæĺ¯ +é¤ ¨ +èĹ ī +çļĦ åIJĦ项 +为 缮æłĩ +çļĦ è®¤çŁ¥ +å½±åĵįåĬĽ çļĦ +夸 å¼ł +佩 æĪ´ +æ±ĩ çİĩ +çļĦ çĪ±æĥħ +æĺ¥ é£İ +æĺ¯ æĪijçļĦ +æ¨ ¹ +åįĬ å°ıæŶ +å±± åİ¿ +å±± 西çľģ +èĢĮ è¿Ļ +æĽ´å¤ļ ä¿¡æģ¯ +è¿ĺ æľīä¸ĢäºĽ +ç²¾ ç»ĨåĮĸ +ç¾İ åѦ +çĶ± æĸ¼ +ä»ħä¾Ľ åıĤèĢĥ +å¾Ī é«ĺçļĦ +åıł åĬł +è¿Ļä¹Ī 说 +å±ķ åĩº +åĽĽ å¤Ħ +ä¸ĩ 家 +æĭĽ åĭŁ +çļĦ 强大 +æĤ£ æľī +å°ı äºİ +ä¹Łè®¸ æĺ¯ +对 èĩªå·±çļĦ +èģĮä¸ļ æķĻèĤ² +æĿ¥ è¿Ľè¡Į +æ¡£ 次 +æīĵ èµ¢ +éĥ½æľī çĿĢ +åº ¸ +è¯Ń æ°Ķ +çĶ² éĨĽ +空 åĨĽ +车 åĨħ +åĽłä¸º ä½ł +å®ŀ æķĪ +æĥħ ä¾£ +åıijè¾¾ åĽ½å®¶ +éķľ åŃIJ +æ¯į å©´ +ä½Ĩæĺ¯ ä»ĸ +积æŀģ æİ¨è¿Ľ +大å¹ħ 度 +çļĦ 女åĦ¿ +é¤IJ æ¡Į +åIJ¬ å¾Ĺ +çļĦ 积æŀģæĢ§ +好 åIJ§ +æĹ¥ æ¶Īæģ¯ +æľī ä»»ä½ķ +æ¯Ĵ åĵģ +æĹ©çĤ¹ åĬłçĽŁ +第ä¸Ģ 天 +å°½ åĬĽ +æł ĸ +主 æīĵ +æĺ¯ä¸Ģ åIJį +çĪĨ æĸĻ +äºĭä¸ļ åıijå±ķ +å¾® åķĨ +äºİä¸Ģä½ĵ çļĦ +çĶŁ çĮª +èĩªçĦ¶ èµĦæºIJ +çŀĦ åĩĨ +è§Ħ模 åĮĸ +并 ä¸İ +èĤ¥ èĥĸ +家 çĶ¨ +大 çĪ· +é¢Ħ åijĬ +æĿ¥ åģļ +éĺ³ åİ¿ +æŀĦ çŃij +é¢ģ å¥ĸ +åİĨåı² æĸĩåĮĸ +æľįåĭĻ æĪĸ +æĢ» åĨ³èµĽ +åıij åŀĭ +æĪij 羣çļĦ +æĽ ¦ +åıĤ ä¼ļ +èĦĨ å¼± +åĩĨ åħ¥ +èħ¹ éĥ¨ +åı¸ 令 +æĤ² åī§ +天 ä¸Ĭ +åı£ ä¸Ń +ä¸ĩ 个 +åѦ ä¸ļ +æıIJ åĢ¡ +两 è¾¹ +大 èĤ¡ä¸ľ +åı¤ éķĩ +è¡Ģ ç³ĸ +çļĦ ç¨ĭ度 +æ£ī èĬ± +åIJİ åı° +å°± åĮ» +æķ´ æķ´ +èĴ ² +çĽĪåĪ© èĥ½åĬĽ +ç± ½ +èĦ « +çľĭ éĩį +家 éķ· +èģĺ çĶ¨ +èµĽ éģĵ +åīį èĢħ +建 èŃ° +å¾ĭå¸Ī äºĭåĬ¡ +èīºæľ¯ åĵģ +æľī èĩªå·±çļĦ +åIJ¦ å®ļ +社 åĽ¢ +åij¨ äºĶ +带 åĪ° +å·¥ä½ľ ä¼ļè®® +èĤ¡ æľ¬ +å¤ĸ åĮħ +家 åħ¬åı¸ +çĽij çĭ± +èĪ Ĭ +åIJį æł¡ +西 æ¹ĸ +è¶ħè¿ĩ äºĨ +åįĹ å±± +ç»Ħ 件 +åĢ¼å¾Ĺ 注æĦı +æĮ£ æīİ +äºĭ 迹 +ç¶ĵ çĩŁ +ç§ij 室 +好 åIJĹ +æ¤ħ åŃIJ +åľĪ åŃIJ +ä½Ĩ 她 +æµģ çķħ +åIJĦèĩª çļĦ +èģĮ åijĺ +è¡į çĶŁ +åħ¨ åľº +æĴ¤ éĶĢ +åį´ 被 +å®ģ éĿĻ +åīį æīĢ +åīįæīĢ æľª +åīįæīĢæľª æľī +主 ä¸ļ +åĮĹ ç¾İ +è¯Ħ å®ļ +åĵģ å°Ŀ +大家 éĥ½åľ¨ +主 å¸ħ +ç»Ĩ å¿ĥ +ä¿¡æģ¯ æĬ«éľ² +çļĦ ç«ŀäºī +éĢĻ樣 çļĦ +ç§ijåĪĽ æĿ¿ +éĩĩ æijĺ +票 æį® +éĢIJ å¹´ +èĭ± è¶ħ +è¡Įä¸ļ åĨħ +人 寿 +åIJİ åĭ¤ +å¦Ĥ æĦı +ç¬Ķ è¯ķ +æ·¡æ·¡ çļĦ +ä¸į èĪĴæľį +ä½ĵ 积 +ä¹Łä¸į è¦ģ +éĿ¢ æĸĻ +æł· æľ¬ +ç¥ ģ +æĮī è§Ħå®ļ +大æ¦Ĥ æĺ¯ +æĥħåĨµ è¿Ľè¡Į +åIJĦ åįķä½į +çļĦ ç¬ij容 +åĩºèī² çļĦ +代表 æĢ§ +çļĦ ç¾İ好 +éĴ ¦ +å¾® çĶŁçī© +è¶Ĭ æĺ¯ +æĸ¹ åı¯ +å¹² èĦĨ +éģĬ æĪ² +çļĦ åħ´è¶£ +éĹ® è´£ +åĽłä¸º æĪij们 +èĢĥ éĩı +çĶŁ çĶŁ +éĺ» åĬĽ +ä¸į åħģ许 +æıIJ è®® +åĩı æĮģ +åıªæĺ¯ ä¸Ģ个 +æĪij æĬĬ +åıijçİ° èĩªå·± +å¢ŀ å¹ħ +å¦ į +èĹĿ è¡ĵ +ä¸Ģ家 人 +åĪĨ 级 +çļĦ æķ°éĩı +è½® èŀįèµĦ +çŃī åĽłç´ł +大 夫 +èģĺ 请 +é£İ æľº +绽 æĶ¾ +ä»»ä½ķ ä¸Ģ个 +éł Ĥ +éĺ¶ çº§ +æĬĬ 她 +è¿Ľ åĨĽ +èĥ½ åģļåĪ° +åŁ¹è®Ń æľºæŀĦ +çī© æĸĻ +ç«¥ è¯Ŀ +æĮĩ导 æĦıè§ģ +éĺ ® +æ·±åħ¥ æİ¨è¿Ľ +主 æľº +æ¸Ķ ä¸ļ +ä¸į æľį +æµĵ éĥģ +è¡Ĺ ä¸Ĭ +ä¾Ŀ 次 +æŶ 段 +æ¢ µ +çļĦ åĸľçĪ± +å¾Ī éķ¿ +åĪĿ 级 +æŀľ æĸŃ +æĬ¢ æķij +é¼ĵ èĪŀ +ä¾Ľ éľĢ +æ·±åħ¥ å¼Ģå±ķ +产ä¸ļ éĽĨ群 +åĻª éŁ³ +åIJ¬ çĿĢ +æ·±åĪ» çļĦ +å¿į åıĹ +çĶµ ç£ģ +强 èĢħ +æ»ĭ åij³ +æĽ¼ èģĶ +åı¯ä»¥ 缴æİ¥ +大 ç±³ +æŃ· åı² +æĶ¿åĬ¡ æľįåĬ¡ +åħ¬ å¼ı +社 群 +éģĵ士 èģĮä¸ļ +ä¹ĭ æĥħ +æµ· æ°´ +æ¼Ķ å¥ı +åºĹ éĩĮ +迹 象 +åıijå±ķ çIJĨ念 +é«ĺ 空 +åij¨ åĪĬ +åĽŀ åĪ°äºĨ +ä¸į éĢĤåIJĪ +åłµ å¡ŀ +åĬ Ī +æ°´ ä¸Ĭ +çĢij å¸ĥ +纳ç¨İ 人 +çĩĥ æ²¹ +å·¥ç¨ĭ é¡¹çĽ® +峡 è°· +æľī éĴĪ对æĢ§ +åľĨ å½¢ +æľ¬ å¸Ĥ +è¿Ļ è¯Ŀ +管çIJĨ èĢħ +ç¡®è¯Ĭ çĹħä¾ĭ +æĬĬ æīĭ +彩 èī² +ä¸Ĭ åīį +夯 å®ŀ +ç¾Ĭ èĤī +å¾Ģ å¹´ +æĵħ èĩª +è¿· 人 +èĪª æ¯į +ç²¾ ç»Ĩ +åľ¨ æĪijçļĦ +åĪĽ æĬķ +麦 åħĭ +æľĪ ç»ı +åĮĹ æµ· +ä¹ĭ æĺŁ +åı¶ åŃIJ +å¸Ĥåľº ç«ŀäºī +è¿Ļ äºĭ +åıĥ èĪĩ +产 åľ° +åĶ ī +åķĨåĵģ æĪ¿ +èĪª è¿IJ +ä¼ĺ å¼Ĥ +ä»ĸ们 æĺ¯ +鼨 æ°´ +è¯į æ±ĩ +åĨľ çĶ° +欧 éĺ³ +çŁŃ 线 +管 ç½ij +æł¹ åŁº +åıªæľī ä¸Ģ个 +éŀĭ åŃIJ +å¸Ĥ å§Ķ书记 +åĪ» æĦı +è¡Į 车 +åıĪ 被 +åı¯éĿł æĢ§ +è´ ± +ä»» åij½ +åºĶ åľ¨ +å°± å¾Ĺ +æľįåĬ¡ ä½ĵç³» +æĶ¿ æĿĥ +åıijè¨Ģ 人 +è¿ĩ å¾Ģ +两 åıª +èĻ½ 说 +éĢģ ä¸Ĭ +ä»Ģä¹Ī äºĭ +æķ£ æĸĩ +æİĮ æݧ +èĸĦ å¼± +ä¸ĭéĿ¢ å°± +主è¦ģ åĨħ容 +å¾Ī éĩįè¦ģçļĦ +å°± 说 +çĻ½èī² çļĦ +éĤ£ä¸ª æŶåĢĻ +ç»ı纪 人 +çļĦ æ¯į亲 +ç¬Ķè®° æľ¬ +åºķ å±Ĥ +è¿ij 代 +解 说 +è²ł 責 +æľĢ大 åĮĸ +åķĨ éĵº +æł¡ åıĭ +æ² ģ +ä¸į åĩºæĿ¥ +éĻ· éĺ± +ç¨ ħ +åħ¬å¸ĥ äºĨ +åĩĢ åĢ¼ +çĽ¸å¯¹ è¾ĥ +ç¬ Ľ +æł¸ ç®Ĺ +åįİ ä¾¨ +æĢ¥ æķij +æĮº 好 +åħĴ ç«¥ +äºĮ èĥİ +åĩº èĩª +åĿ Ł +æīĭ ä¸ĭ +å± ¡ +åĪĽéĢł æĢ§ +ä¸¥æł¼ æĮīçħ§ +åĨį åİ» +举 缣 +人 æµģ +äºĨä¸Ģ 声 +å°ıæŶ åīį +è´µ æĹı +éľ ĸ +ä¹Łæĺ¯ éĿŀ常 +éĢ ± +çľĭäºĨ çľĭ +ç¹ģ æ®ĸ +èĩ³ æѤ +é¢Ħ å¤ĩ +å¾Ī æĺİæĺ¾ +æ¼Ķ èīº +åĿIJ çĿĢ +ä¿Ħ åĨĽ +åľ¨ è¿ĩåİ» +ä¹ĭ äºĭ +æĬĵ èİ· +åĿIJ ä¸ĭ +çĶ± ä¸ŃåĽ½ +ä¹Ł å¼Ģå§ĭ +çŃĶ å¤į +åŀĥåľ¾ åĪĨç±» +éĴĵ é±¼ +åIJĦ 種 +缸 éģĩ +ä¸įåģľ çļĦ +æī¹ éĩı +éĩįè¦ģ ä½ľçĶ¨ +å§Ķ å±Ī +åħŃ å¹´ +ä¸ĥ åįģ +ä¹ĭ æĪĺ +é£İéĻ© 管çIJĨ +éŁ³ æ¨Ĥ +è¡ĮæĶ¿ å¤Ħç½ļ +æľ¬ äºĭ +æĴ° åĨĻ +èģļ åIJĪ +éĢĤ æŶ +æIJ¬ 家 +ç¢İ çīĩ +缼 å®´ +ç®Ģ æ´ģ +åı¬ éĽĨ +ç®Ģ åĮĸ +åĮĹ京 æŶéĹ´ +第ä¸ī å±Ĭ +æĿ¥ åĽŀ +常çĶ¨ çļĦ +京 æ´¥ +京津 åĨĢ +梦 å¹» +è¯ķ è¡Į +æľº åºĬ +åĪ° æľĢåIJİ +åĬ© æīĭ +åĪĨ 彩 +åĩº åĵģ +åĪ¹ 车 +åIJ¯ åıij +侧 éĿ¢ +æ¯ı å½ĵ +缸åħ³ è§Ħå®ļ +ä¸ĸ 人 +è´Ń 车 +å¿ĥ 缮 +å¿ĥ缮 ä¸Ń +äºĶ éĩij +è¿ĺ è®°å¾Ĺ +ä¾Ŀ çĦ¶æĺ¯ +æıIJ æ¡Ī +çĶµåķĨ å¹³åı° +åģļ åĪ°äºĨ +æĿľ ç»Ŀ +å®ī åįĵ +ä¸ĸçķĮ åIJĦåľ° +åīį éĢĶ +æ´Ĺ åĩĢ +å¥ĭ åĬĽ +åŁİå¸Ĥ 建设 +å¤ļ åĬŁèĥ½ +ä¼ļ éĢłæĪIJ +åıijå¸ĥ ä¼ļä¸Ĭ +究 ç«Łæĺ¯ +åĪĨ 红 +çŁ¥ èŃĺ +éĿ¢ æĿ¿ +æĹł 声 +æĢ¥ éľĢ +失 çľł +çĪ¸ å¦Ī +äº Ĥ +åħ¨ æĻ¯ +ç»ıåħ¸ çļĦ +åī§ ä¸Ń +é¢Ĩ导 ä¸ĭ +åħļ åĨħ +åħ¥ ä¾µ +æĭī æĸ¯ +ä¸Ģ å¹ķ +åĬł ä¹ĭ +èĤ Ĩ +èĭ± æł¼ +èĭ±æł¼ åħ° +å·§ åħĭ +å·§åħĭ åĬĽ +ä¸Ģ å¿ĥ +èģ Ĥ +å¾Ģå¾Ģ æĺ¯ +管çIJĨ å±Ĥ +çĻ» åħ¥ +建ç«ĭ èµ· +建 åĽ½ +åŃIJ 宫 +åºĶ ä»ĺ +æİ¢ 究 +第ä¸Ģ ä½į +ä½Ļ 家 +çŃī æ´»åĬ¨ +æīĢ èĩ´ +è¾ĥ å¿« +æĺ¯ éĿŀ +æıIJ åIJį +äºĮ èĢħ +åıªåī© ä¸ĭ +åħ¶ä¸Ń åĮħæĭ¬ +ç¼ĸ ç¨ĭ +çł´ ç¢İ +ä¸Ń 举 +å·¥ä½ľ æĬ¥åijĬ +çŃ¾ åIJį +éħĴ ä¸ļ +çŁ¥ æĻĵ +çĥŃ å¿ĥ +éĿŀ åĩ¡ +èIJ¥ä¸ļ æī§ +èIJ¥ä¸ļæī§ çħ§ +人大 代表 +ä¸Ģ个 æĸ°çļĦ +å¨ģ æµ· +éĤ£ 人 +涨 ä»· +æ¶Ī çģŃ +éļ¾ å¿ĺ +ç¶ĵ é©Ĺ +åı£ è¢ĭ +ç³» æķ° +æĸĩ ä¸Ń +好 转 +æĸ° 鼶åĶ® +讲述 äºĨ +å¼Ģ çĽĺ +çķĻ ç»Ļ +æħ¢æħ¢ çļĦ +æĤ² 伤 +æľ¬ æľŁ +äºĨ å¤ļå°ij +è¿Ļ 让 +åIJĮ çŃī +æ¸ħ æĺİ +个 åŁİå¸Ĥ +æºĸ åĤĻ +åĩłä¹İ æĺ¯ +强 åĬĽ +ä¿ ¯ +æ°´ 稻 +åĽºå®ļ çļĦ +æł¸ åĩĨ +说 æľį +顯 示 +è¿Ļ å¥Ĺ +æĻºæħ§ åŁİå¸Ĥ +å±ĭ 顶 +ä¸į æĿ¥ +çĶŁ é²ľ +çŁ¥ æĥħ +æĬķ 身 +åijĬè¯ī æĪij们 +ä¸ī åĽĽ +ä¸ĩ ä¸Ģ +è¾Ĩ 车 +为 ä¹ĭ +åĪ° æŶåĢĻ +è¿Ļ æīįæĺ¯ +åIJį çīĮ +åºŁ æ°´ +åݻ年 åIJĮæľŁ +å¹´ éĻIJ +éģĭ åĭķ +åıĮ çľ¼ +è¦ģ ç´§ +对 çŃĸ +åľº é¦Ĩ +çĻ¾ ç§ij +è¶Ĭ éĩİ +å¯Į åIJ« +大å¤ļæķ° 人 +æľĢ å°ij +åı¬ åĶ¤ +åħ¸ èĮĥ +åĨľ æľº +æŃ£ æĸĩ +åºĶçĶ¨ äºİ +æ·± èĢķ +ä¿ Ń +ä»Ģä¹Ī ä¸ľè¥¿ +å¥Ĺ é¤IJ +å½ĵ éĢī +å·¦ æīĭ +è°ĥ çIJĨ +æĻļ é¤IJ +éļ¾ åħ³ +åĩŃ è¯ģ +çĪ± 人 +æĮĩ è´£ +è´£ ç¼ĸ +çļĦä¸Ģ 款 +éĵ ² +åįģ 个 +èĢ » +æľįåĬ¡ åķĨ +åľ° çĭ± +è¿ŀ å¿Ļ +åĽ° æĥij +çļ ĵ +ä¸į åIJĥ +çİ°åľ¨ å·²ç»ı +çĽĺ çĤ¹ +ä¸įåģľ åľ° +管çIJĨ 模å¼ı +è¿Ļ 段æŶéĹ´ +æ¤ ° +礼 åĮħ +æµģ 转 +æī« çłģ +éĽĨä¸Ń åľ¨ +æ±Ĥ åĬ© +åįĬ 个 +å¿«éĢŁ å¢ŀéķ¿ +å¾Ģ ä¸ĭ +è¯Ħ åĪĨ +å°± æĥ³ +åķĨåĬ¡ éĥ¨ +æľī éĹ®é¢ĺ +èİ· åĪ© +æ¯Ľ çĹħ +æĦŁ åºĶ +èī¯ æĢ§ +åĪĨ æѧ +åĨ ī +æĪij们 çİ°åľ¨ +è¦ģ åĬłå¼º +å·§ å¦Ļ +èŀº æĹĭ +åĪĩ æį¢ +çĭ Ħ +顺 çķħ +å°¤åħ¶ æĺ¯åľ¨ +èĬĿ 麻 +éļ¾ è¿ĩ +æĹĹ å¸ľ +å¤į åį° +å¤įåį° 件 +å¿ħ éľĢ +对å¤ĸ å¼ĢæĶ¾ +éļ¾ åıĹ +åİŁæĿ¥ æĺ¯ +ç®Ĺ äºĨ +é«ĺ å±± +离 èģĮ +çµĦ ç¹ +çµĦç¹ Ķ +å±ģ èĤ¡ +çĻ¾ 家 +éģĩ ä¸Ĭ +æĺĶ æĹ¥ +ä¸į 容 +çĽij管 éĥ¨éŨ +主 æĦı +æµģ åŁŁ +è·Į å¹ħ +èĩ³ ä¸Ĭ +åĪ« 说 +æĺ¯ æ¯Ķè¾ĥ +å®ıè§Ĥ ç»ıæµİ +å¸Ĥåľº 主ä½ĵ +污æŁĵ çī© +æķij æ²» +丰 æĶ¶ +åŃĺ æĶ¾ +åĩ Ħ +éĩij å±± +æį¢ äºĨ +ä¸ĵ 人 +éĹľ æĸ¼ +æĹ¢ è¦ģ +åĽ½ 足 +éļ ĭ +åıį åĩ» +èµ· 身 +åħĪ æĺ¯ +å¸ĮæľĽ èĥ½å¤Ł +åĪ¶ 订 +åºĹ éĿ¢ +åĸ Ģ +æķĻ ä½ł +éĻį 温 +åĬĽ æ±Ĥ +ä¸ī çĻ¾ +çī© ä»· +丢 失 +å¢Ļ ä¸Ĭ +éĥ¨ 份 +æł· æĿ¿ +ä¹ĭ æĦı +ç½ij å°ıç¼ĸ +ä¸ĸ ä¸Ĭ +è°ĥ è¯ķ +污æŁĵ éĺ²æ²» +å½± éĻ¢ +å®Įåħ¨ åı¯ä»¥ +éĢļ åħ³ +ä¹īåĬ¡ æķĻèĤ² +没æľī åĬŀæ³ķ +èĢ ¿ +å¦ ³ +æĹł æĥħ +å¾Ĺ çĽĬ +å¾ĹçĽĬ äºİ +æľŁ çĽ¼ +娱ä¹IJ åľº +çĶ² æĸ¹ +ä¸Ģ æ±½ +çĹ ° +çĸij ä¼¼ +æĸ°æµª å¾®åįļ +强 è¡Į +å½ĵ ä»ĸ +èĥ º +çĶ¨æĪ· æıIJä¾Ľ +åĮº å§Ķ +æĦ¿ æĻ¯ +æĬĺ æī£ +失 踪 +è¿« åĪĩ +åŃĹ æ¯į +åĴ ¯ +èªį èŃĺ +ä»Ģä¹Ī æĦıæĢĿ +çĽĴ åŃIJ +å½ķ éŁ³ +建设 å·¥ç¨ĭ +ä¸ļ ä½Ļ +å®ŀè·µ æ´»åĬ¨ +羣 空 +çĤ ĸ +åľ¨ è·¯ä¸Ĭ +主è¦ģ åĮħæĭ¬ +该 æĢİä¹Ī +æĢ» æľī +æĢ§ æĦŁ +æ°ij èĪª +å¼Ģ åºĹ +欺 éªĹ +çªģ åĩ» +缺 失 +æī§ ä¸ļ +åľ° éģĵ +并 æĹł +æ°ij åĬŀ +ç»Ħç»ĩ çĶŁæ´» +æĪij å¦Ī +è¨ĺ èĢħ +管 åĪ¶ +æī¾ 个 +èĹ » +çĤİ çĹĩ +äºĴ åĬ© +æµıè§Ī åĻ¨ +çݩ家 æĿ¥è¯´ +éĻįä½İ äºĨ +è£ Ķ +æĮ£ éĴ± +åķĨ æľº +æĶ¹ è£ħ +æµģ 浪 +æĶ¿ æ³ķ +èĢģ 头 +çĶŁäº§ åĴĮ +ç© Ĺ +亲 çĪ± +亲çĪ± çļĦ +å±¥ èģĮ +åŁİ éĩĮ +ç»Ĩ åĪĨ +åĬ³åĬ¨ åIJĪåIJĮ +åľ¨ æĹ¥æľ¬ +å¨ģ å°Ķ +åį« è§Ĩ +éĢ£ çµIJ +çĿĢ éĩį +æĬĺ 磨 +åĽ¾ 为 +çľ · +å·¥ åºı +æĵ ģ +æĵģ æľī +ç½ijç«Ļ åľ°åĽ¾ +çļĦä¸Ģ 大 +ç»Ħç»ĩ å®ŀæĸ½ +æĬĽ å¼ĥ +åĴĮ æĶ¯æĮģ +æ³ķ åĪĻ +浪 æ½® +çİ° æľīçļĦ +åĩł çİĩ +为 客æĪ· +åįģ ä¸ĩ +è ¹Ħ +çªģåĩº éĹ®é¢ĺ +åıĥ åĬł +éĥ½ä¼ļ æľī +çĽ ¤ +è°ģ éĥ½ +æīĭ åĬ¨ +缴 è¾¾ +çĤ¹ å¤ļ +éĺ¶ å±Ĥ +ä¸į ä½³ +éĤ£ 段 +滨 æµ· +æĺ¯ åĽ½åĨħ +æĪij å¸ĮæľĽ +åIJĽ åŃIJ +è§Ĥ éŁ³ +åģļ é¥Ń +æ±½ è»Ĭ +åħ³ ç¨İ +çľ¼åīį çļĦ +æ°´ éĿ¢ +èĢ³ æľº +追 踪 +æݨ éĢģ +éĴ± åĮħ +æģ¶ å¿ĥ +æµ· åŁŁ +å· į +å¼Ģ æĿ¥ +表 æĢģ +仪 表 +å¹³ åİŁ +åįģ å¤ļå¹´ +ä¹Ł æĹłæ³ķ +åħ¼ 顾 +è¡£ æŁľ +æł½ åŁ¹ +æĪ¿ æºIJ +设ç«ĭ äºĨ +ä¸ĩ åIJį +æķ° é¢Ŀ +è¦ģ åĿļæĮģ +åIJīæŀĹ çľģ +请 èģĶç³» +ç»ıåİĨ è¿ĩ +çļĦ æľ¬è´¨ +åħ¥ éŨ +æľ¬ æ¡Ī +çİĩ è¾¾åĪ° +åı° éĺ¶ +éĴ ŀ +æĪij èĥ½ +èݲ èĬ± +éĴ ł +ä¸Ģ äºĭ +åİŁ æľīçļĦ +æ¯ı åĢĭ +æ¯Ķäºļ 迪 +æ£ĭçīĮ 游æĪı +ä¸įä¼ļ æľī +å½Ĵ æĿ¥ +äºĶ çĻ¾ +è¿ĩ é«ĺ +鼷 è¾¾ +ä¸Ģèµ· åİ» +æķĻ 导 +å°± è¯Ĭ +å°± å¾Ī +ä¸įåIJĮ äºİ +ä¿ º +å¸ĸ åŃIJ +æĶ¿åįı å§Ķåijĺ +çĸ«æĥħ å½±åĵį +åĪĨ è£Ĥ +为ä»Ģä¹Ī ä¼ļ +äºĶ æĺŁ +å°ij åĦ¿ +æĬ¢ éĻ© +梦 è§ģ +è®°èĢħ éĩĩ访 +å±± è·¯ +æĪij 个人 +æ²Ļ 滩 +è¹ Ń +æĶ¹ è®Ĭ +æĸ°åŀĭ åĨł +æĸ°åŀĭåĨł çĬ¶ +åĮ» æĬ¤ +åĮ»æĬ¤ 人åijĺ +æµ· å°Ķ +åħ³äºİ æĪij们 +éĻ¤ å¤ĸ +åº ļ +宣 åijĬ +ä¸ī åįĥ +æ¦ ¨ +ç§ijæĬĢ 大åѦ +ä¸ĥ åħ« +顺 åºĶ +çĪ¸çĪ¸ å¦Īå¦Ī +éĢī åıĸ +åī§ çĥĪ +乡æĿij æĹħ游 +积æŀģ æݢ索 +表çİ° 为 +å¾Ī æ¸ħæ¥ļ +大 åĨĽ +æĿ¥ çĶµ +å¥Ĺ æĪ¿ +çİ° è¡Į +享 åıĹåĪ° +çľĭ çĤ¹ +åĽºå®ļ èµĦ产 +以 人为 +以人为 æľ¬ +ä¸į å®Į +éĻį 鼨 +åģļçļĦ äºĭæĥħ +并 äºİ +顽 强 +èĢ ¸ +åĺ´ å·´ +缸åħ³ ä¿¡æģ¯ +æĪij 没 +æĪĺçķ¥ æĢ§ +æĢĿ 念 +åĪĺ å¤ĩ +åĬ© æĶ» +é£İ è²Į +éĿ¢å¯¹ éĿ¢ +积æŀģ å¼Ģå±ķ +çĸĹ æķĪ +çľĭ 书 +缺 åı£ +åĽ½æ°ij ç»ıæµİ +使çĶ¨ æĿĥ +éģ¥ è¿ľ +å¡« è¡¥ +第ä¸ī 人 +åįĬ å¤ľ +æѦæ±ī å¸Ĥ +æĪij åıijçİ° +ä¼ĺæĥł æĶ¿çŃĸ +é£İ åı£ +å°± ä¸įèĥ½ +为 主è¦ģ +æµģ åĩº +å´ĩ æĭľ +并 ä¸įèĥ½ +é«ĺ ä¸ī +ä¸ĸçķĮä¸Ĭ æľĢ +æĥ³ å¿ħ +åħ¶ æīĢ +åĢĻ éĢī +åĢĻéĢī 人 +ä¸į çĪ± +åī¯ ä½ľçĶ¨ +人æ°ij æĹ¥æĬ¥ +æĪij ä¸įæĺ¯ +å®ŀ çī© +çĶµ åİĤ +ä¹Ł ç®Ĺæĺ¯ +æľī éĹľ +æľī èĥ½åĬĽ +æĮĤ åľ¨ +çľ¼ ä¸ĭ +约 ç¿° +å°ı åѦçĶŁ +èµ· åĪ°äºĨ +å·¥ 夫 +åIJĮ å¿ĥ +åĿ¦ è¨Ģ +çł Į +åıijæĮ¥ äºĨ +èģĮä¸ļ éģĵå¾· +è¿ĻäºĽ å¹´ +念 头 +èĢģ é¼ł +åħ¨ èµĦ +åħ¨èµĦ åŃIJ +ä¸Ģ åij³ +å¤ļ ä¸ĩåħĥ +æł¼ æľĥ +éķ¿ éĢĶ +带 èµ° +èĭ± 寸 +æĸĩ ä½ĵ +对 ä»ĸ们 +åĵŃ äºĨ +å¡« æĬ¥ +çīĪæĿĥ 声æĺİ +çĶµ 线 +è´Ńçī© ä¸Ńå¿ĥ +饱 满 +ä½İ 头 +强 è¿« +ä¿Ŀ æ´ģ +欧 åĨł +缸 è¿ŀ +认 è´Ń +çģ« æĺŁ +é«ĺ å°Ķ +é«ĺå°Ķ 夫 +èij« èĬ¦ +æłĩ 注 +çļĦ çIJĨæĥ³ +æł¸ éħ¸ +æł¸éħ¸ æ£Ģæµĭ +åĬ ī +ä¸ĢèĪ¬ æĺ¯ +æĢĿ ç´¢ +轨 迹 +çĥŃ å¸¦ +éĻ £ +åĩĨç¡® æĢ§ +æĪ´ çĿĢ +åľ¨ çĶŁæ´»ä¸Ń +æīĢ èĥ½ +æľ¯ åIJİ +带 ä½ł +ç¥ ł +æ®ĭ éħ· +ä¹Ł åıªæĺ¯ +çĶ³ è´Ń +举åĬŀ äºĨ +æľī æĦıä¹ī +æĹº 缼 +åľ¨ 網 +åľ¨ç¶² è·¯ä¸Ĭ +å¾Ī大 ç¨ĭ度 +管 è¾ĸ +çĸ«æĥħ æľŁéĹ´ +触 æij¸ +éĺ¶æ®µ æĢ§ +ä¼ļ è§īå¾Ĺ +çļĦ çĶ»éĿ¢ +æİ¥åıĹ äºĨ +表达 äºĨ +éĤĵ å°ı +éĤĵå°ı å¹³ +åħļ é£İ +åħļé£İ å»īæĶ¿ +åķĨ åѦéĻ¢ +åħij æį¢ +é£Łåĵģ èį¯åĵģ +éĿŀ常 好çļĦ +çľ ¯ +纳 ç±³ +åĬ¨ æijĩ +åĽŀ éģ¿ +çľĭ èijĹ +款 项 +åħ« å¹´ +åģļ 个 +æĸĩ æ¡£ +éĩijèŀį ç§ijæĬĢ +åħ¶ä¸Ń æľī +äºĨä¸Ģ ç³»åĪĹ +æĹĹèĪ° åºĹ +称 èµŀ +éĽ¢ éĸĭ +åĪ¶ åĨ· +家 éŨåı£ +åįģ å¤ļ +ä¼´ ä¾£ +çľĭ çĹħ +æĭī çĿĢ +æī Ĵ +çĸ² æĥ« +å°ijæķ° æ°ijæĹı +åĽ¾ å½¢ +è½ § +å¢ŀ éĩı +饲 åħ» +çģ« å±± +æ¯ı 个æľĪ +ä½ľä¸º ä¸ĢåIJį +è½´ æī¿ +æĸĩ 书 +ç¼ ķ +åħ·ä½ĵ æĥħåĨµ +çĹĽ çĤ¹ +缴 éĶĢ +å¡ Ĭ +ä¹Ł æľĥ +çĥŃ æ½® +å¹³ æ°ij +æ¼ĶåĶ± ä¼ļ +æķĻ çłĶ +éĢĥ éģ¿ +ä¸Ģ è´¯ +å°± è¶Ĭ +å®ŀ å®ŀåľ¨ +å®ŀå®ŀåľ¨ åľ¨ +ä¹łè¿ijå¹³ æĢ» +æº º +å¿ĥ åºķ +éķ¿ å¾ģ +媽 媽 +第ä¸ī 次 +åĩº æ¼Ķ +çĭĢ æ³ģ +å°Ķ æĸ¯ +代çIJĨ åķĨ +çĨ ı +çļĦ 对象 +çĶµ éĩı +è¡Į åĪĹ +åĽ½ 人 +è·ij äºĨ +åįĶ åĬ© +èIJ¥ è¿IJ +å¸Ī åħĦ +æ¦ ® +æĥ³ åĥı +æĢ§ 强 +ç§ijåѦ çłĶ究 +延 å®ī +ä¸¥æł¼ èIJ½å®ŀ +é¢Ĩ ä¼ļ +缸 å·® +è·¯ 人 +çĶ « +æľī ä»·åĢ¼ +æľīä»·åĢ¼ çļĦ +ç¾İ åĽ¢ +æ°ij主 çĶŁæ´» +æĪij æīį +ç¾İåĽ½ 人 +æ°Ķ åij³ +åıį å°Ħ +çļĦ åĨ³å¿ĥ +大 è±Ĩ +交 代 +è¿Ľ åĩº +åıį æĬĹ +æĮĩ çļĦæĺ¯ +ä»· ä½į +è¿Ľ é©» +ä¸Ĭ çĻ¾ +ä½į åĪĹ +ä¸ŃåĽ½ ä¼ģä¸ļ +çļĦ好 å¤Ħ +主 ç¼ĸ +æ±½ æ²¹ +ä½Ĩ æĪij们 +æĢİä¹Ī çľĭ +é»Ħ å±± +å¤ļ åªĴä½ĵ +åIJİ åį« +èİ·å¾Ĺ æĽ´å¤ļ +åĬ¡ å¿ħ +为 å¥ijæľº +é¦ĸ 饰 +ä¸ĩ åįļ +è¶ĬæĿ¥è¶Ĭ 大 +ä¸ĵ项 è¡ĮåĬ¨ +å¥ĭ è¿Ľ +ä»į çĦ¶æĺ¯ +è´¨ æĦŁ +å¦Ĥæŀľ ä¸įæĺ¯ +ç«Ļ èµ·æĿ¥ +ä¹¾ éļĨ +åı¯æĢķ çļĦ +å¯Į è´µ +æ¸ħ ç®Ĺ +åIJij ä¸ĭ +åĢ ļ +çļĦ çŃĶæ¡Ī +èĪ¹ ä¸Ĭ +çļĦ羣å®ŀ æĢ§ +çŃī åĬŁèĥ½ +åĸľ åī§ +å¨ģ åĬĽ +æĸ° é¢ĸ +æł¸ çĶµ +æĬ¥ éĶĢ +æķħ 乡 +ä¼´ éļı +éŀ Ń +å¦Ĭ å¨ł +åĪĨ åĮĸ +æľī å¾Ī大 +æĢİä¹Ī 说 +æĻĤ 代 +产 åĩº +ä»ĭç»į 说 +å¤ĦçIJĨ åĻ¨ +èĨ¨ èĥĢ +åī¯ å¸Ĥéķ¿ +çļĦ 妻åŃIJ +æł· åĵģ +åIJĮæ¯Ķ ä¸ĭéĻį +åħĥ å·¦åı³ +çĶ¨ èĩªå·±çļĦ +é«ĺ éĽĦ +æĺ¥ æĻļ +ä¹Ł æľīå¾Īå¤ļ +çľ¼ çIJĥ +æķ£ æŃ¥ +ä»ĸ们 éĥ½ +第ä¸Ģ 家 +åĬŀ 好 +å®ī éĺ² +ä¸Ģ ä¸ĩ +åľ¨ éĩĮéĿ¢ +éŁ³ é¢ij +åı£ åı· +ä¸Ģ è¶Ł +ç¦ı çī¹ +é³ ŀ +æĥĬ èī³ +æĸ° å¨ĺ +绿èī² åıijå±ķ +ä¸Ń å¼ı +ä¹Ł åıªæľī +çİ° 身 +åı¯ ä¾Ľ +æ¯ı ä¸Ģ个人 +第ä¸ī èĢħ +åľ° å½¢ +éĴ¢ ç»ĵæŀĦ +çĽijçĿ£ æ£ĢæŁ¥ +åı« æĪij +èĩ´ æķ¬ +æ´Ĺ æīĭ +ä¸ĭ è°ĥ +康 çĨĻ +æĪIJ交 éĩı +ä¹Ł æĪIJ为 +åħī æ»ij +å®Įæķ´ æĢ§ +çģ ¼ +網 éłģ +éķ¿ 寿 +éģ© çĶ¨ +çļĦä¸Ģ 项 +çŀ© 缮 +æĬĬ èĩªå·±çļĦ +éĵ¶è¡Į åį¡ +å°± å¿ħé¡» +ç¾İ çĻ½ +éŀį å±± +æľ¬ é¢Ĩ +ä¸Ģ ç¢Ĺ +æīĵ æ³ķ +æĤ¨ 好 +对 åŃ©åŃIJ +æĬ¥éģĵ 称 +ä¼ł åĩº +大 èĩ£ +ç¬ ĭ +çĽ ı +é¾ ļ +缴 线 +æĻº åºĵ +秣 车 +é£İ åij³ +çľĭ ä¸Ģä¸ĭ +æݨ éĶĢ +éĥ¨ éĥ¨éķ¿ +è´¨éĩı åĴĮ +åĪĬ çĻ» +å·¥ä¸ļ åĮĸ +çİĩ 为 +鼶 件 +硬 åĮĸ +ä¸Ĭ åįĥ +ç»ıéªĮ åĢ¼ +å¹³ è¡Į +声 éģĵ +æľįåĬ¡ è´¨éĩı +çĶŁ çĶ¢ +æľĢ 容æĺĵ +ä¸Ģ æŀļ +å¹´ æĬ¥ +åħ¬ ç½ij +åħ¬ç½ij å®ī +åħ¬ç½ijå®ī å¤ĩ +çļĦ èĥ½éĩı +å®ŀéĻħ è¡ĮåĬ¨ +è¦ģ ä¸įè¦ģ +æĹ¥æľ¬ 人 +èĢ¶ 稣 +ç¼ĸ åī§ +æ¶ © +åį° å°¼ +ä¸Ĭä¸ĭ 游 +åĩł åı¥ +ä¸Ń éĵģ +ç°¡ åĸ® +èĩª 带 +çĶŁ äºİ +ä¸Ģ åı£æ°Ķ +åĭ¤ å¥ĭ +éĻį ä»· +å±ķçİ° äºĨ +å¸ĥ æĭī +ä¼ļ éĢīæĭ© +çļĦ ç»ıåħ¸ +好 æľĭåıĭ +车 éģĵ +æķ´ åĢĭ +åľ ĵ +éķ¿æľŁ 以æĿ¥ +æĬķ å½± +çļĩ åĨł +è¿ĩ 大 +åijĬè¯ī ä»ĸ +ä¼ģä¸ļ æıIJä¾Ľ +æĬ½ 象 +éĢĤ 度 +çļĦ 女åŃ© +èµ· ä¼ı +çļĦ åĬŁæķĪ +ä¸ĵ项 æķ´æ²» +åı¯ éĢļè¿ĩ +ä¸įåIJĮ ç¨ĭ度 +å¼Ĥ è®® +åĩĢ èµĦ产 +åij Ĺ +ä»Ģä¹Ī åij¢ +å·¡ éĢ» +è¸ı ä¸Ĭ +ä½Ĩ å®ĥ +ç²¾ 度 +管 å±Ģ +第ä¸Ģ åIJį +åĨħ åŃĺ +æijĨ åľ¨ +åī© ä¸ĭ +主ä½ĵ 责任 +çĤ¹ åįĬ +以 èĩ³äºİ +åħ»èĢģ ä¿ĿéĻ© +æĦŁåıĹ åĪ°äºĨ +çŁ¥åIJį çļĦ +å¯Į 豪 +妥 åĸĦ +åŃĻ åŃIJ +éĵ Ĥ +说 èĩªå·± +让 æĤ¨ +æķ° æݧ +çļĦçľ¼ åħī +注 éĶĢ +çļĦ çģµéŃĤ +è¿ĺ ä¸įéĶĻ +éĹ® ä»ĸ +èĩªä¸» çłĶåıij +èĵ ĭ +ç´« èī² +åĽ½å®¶ å®īåħ¨ +è¾½å®ģ çľģ +ä¹Ł æ¯Ķè¾ĥ +ç¾İ èĤ¡ +ä¸įç¡®å®ļ æĢ§ +å¿ĥ 头 +æĪ ³ +级 åĪ«çļĦ +论 è¿° +çļĦ åĽŀçŃĶ +ä¿Ŀè¯ģ éĩij +çŃī è¡Įä¸ļ +幸ç¦ı æĦŁ +æѧ è§Ĩ +æľº 票 +æ´¾ 人 +èĩ´ åij½ +åĺ´ è§Ĵ +æĸ°éĹ» ä¸Ńå¿ĥ +æĶ¾å¼ĥ äºĨ +å®ľ å±ħ +åĨĻ ä¸ĭ +éĹ® çŃĶ +è¿ĻéĩĮ æĺ¯ +å¤ļ åľ° +åĮºåŁŁ åĨħ +åīµ æĸ° +çľĭ ä»ĸ +æī§æ³ķ 人åijĺ +åĬ¨ æľº +éŁ³ åĵį +çļĦ åij½è¿IJ +顶 éĥ¨ +åĵ Ł +éĥ½ æľĥ +æīĵéĢł æĪIJ +æĦı åĽ¾ +çļ ĸ +åĢĴ åħ¥ +å·´ èIJ¨ +åĬ© åѦ +å¤į åı¤ +åIJ¯ çĶ¨ +åĽ½éĻħ å¸Ĥåľº +åĤ¨ èĥ½ +é»ijé¾Ļæ±Ł çľģ +ä¹ĺ 车 +è¿IJåĬ¨ ä¼ļ +ä¿Ŀ åĪ© +çŁ³ æĿIJ +çµ ® +çĤĴ ä½ľ +çļĦ ä¿¡ä»» +å°± æĪIJäºĨ +åı¯ è§Ĥ +çļĩ ä¸Ĭ +è¿Ļ åĩłå¤© +ä¸Ģ éĶ® +åĨ· åĨ» +ä¿Ŀ åį« +æł¸ æ¡ĥ +åIJĪä½ľ åħ³ç³» +éĢģ åĩº +æĹĹ ä¸ĭçļĦ +åľ¨ ä¹İ +为 广大 +åįĪ é¤IJ +ä¸ĵ 访 +æĪĸ å°Ĩ +éĿĴå²Ľ å¸Ĥ +å¥Ķ è·ij +æĹ¥ æĬ¥éģĵ +å¥ij åIJĪ +æĸ° æĺ¥ +ä¸į å°ıå¿ĥ +两 ä¸ī +æĦıæĢĿ æĺ¯ +åĨ· èĹı +çļĦ çĹĩçĬ¶ +æĢ§ åij½ +è¶ħ æłĩ +å¯Ĩ 碼 +ç§ijæĬĢ èĤ¡ä»½ +äºĨä¸Ģ æī¹ +çĿ£ å¯Ł +åªĴ ä»ĭ +å°Ħ æīĭ +ä¿® åħ» +çīĩ åĪ» +éĢĤåIJĪ èĩªå·± +åıªè¦ģ æĺ¯ +åIJĥ è¿ĩ +éĩij éĵ¶ +缴 å±ŀ +åѦ éĹ® +åİĭ åĪ¶ +çªĹ å¤ĸ +æĶ¶ åĪ°äºĨ +åħ¨åĽ½ 人大 +ä½Ĩæĺ¯ å¯¹äºİ +åľ¨ æķ´ä¸ª +çļĦ èĥĮåIJİ +åĩıå°ij äºĨ +åıį èħIJ +åıįèħIJ åĢ¡ +åıįèħIJåĢ¡ å»ī +æĹ · +åĪĨ æľŁ +åľ¨ æ·±åľ³ +æīĵ çĿĢ +æī« ä¸Ģ +æī«ä¸Ģ æī« +æĶ¿åºľ éĥ¨éŨ +æİ¥ è¿ŀ +å±ŀäºİ èĩªå·± +åŃIJ å¼¹ +åIJĮæł· æĺ¯ +æĢ» åħ± +车 ä¼ģ +æ¢ ĵ +åħ¬ é¡· +åıij 声 +éĴ Ľ +èµ°åĬ¿ åĽ¾ +主 èIJ¥ +åĸ Ķ +æķ°æį® åĪĨæŀIJ +ä¸į è¿ľ +æľī åIJį +æľīåIJį çļĦ +åģ¿ è¿ĺ +å¾Ī ä½İ +è®ĵ 人 +èĿ ī +é«ĺ è´µ +å°ij 许 +æ° Ł +å¹ ¢ +亲 æĥħ +è¿Ļ件 äºĭæĥħ +çĶ¨ é¤IJ +缸åħ³ æĸ°éĹ» +å°± åºĶ该 +ç»Ī çĤ¹ +æĺ¯ å¤ļå°ij +çĻ» åľº +è¯ķ 管 +è¯ķ管 å©´åĦ¿ +åģļ 大 +åģļ大 åģļ强 +çļĦ ä¾ĭåŃIJ +åħ« 个 +æĺİ æĹ¥ +çĤ ³ +èµ° åİ» +éģ º +å¢ © +ä½ĵä¼ļ åĪ° +åĴ ı +ä¸ĭ è¾¾ +å¤į åıij +追 éĢIJ +æīĵ åĵį +çļĦ éļ±ç§ģæ¬Ĭ +åħ·æľī ä¸Ģå®ļ +è¿Ļä¹Ī å¤ļå¹´ +æłij æŀĹ +æľĢ éķ¿ +åIJĮ èĥŀ +åħī æ³½ +åŁŁ åIJį +æĮĩ åIJij +åıĹ害 èĢħ +æłij èĦĤ +æľīå¤ļ 大 +大 éĿ¢ç§¯ +æĹł ç¼Ŀ +æĶ¹ æŃ£ +æĽ´å¤ļ çļĦæĺ¯ +æľŁ æľ« +æŃ ¼ +ä¹ī ä¹Į +éĤ£ ä½ł +çļĦ 第ä¸Ģ个 +èĮ µ +å° § +èį « +ä¸įä»ħ åı¯ä»¥ +æ¶Į çİ° +æĢ» éĿ¢ç§¯ +æĸ°éĹ» åıijå¸ĥ +æ°ij çĶ¨ +å°± 读 +æīĵ è´¥ +å¤ĸ è¯Ń +æĪij们 ä¸Ģèµ· +é¢Ħ å®ļ +çĥ¹ 饪 +æľĢ 主è¦ģ +æľĢ主è¦ģ çļĦ +çīĮ çħ§ +åĽł åħ¶ +ä½İ ä¸ĭ +ä¼ļ åIJĮ +è§ģ 解 +éĹ´ éļĶ +æķĻ ç¨ĭ +å° ī +å¸Ĥ ä¸Ńå¿ĥ +åħ³éĶ® æĺ¯ +æµ· åįĹçľģ +çī¹åĪ« æĺ¯åľ¨ +ä¸ŃåĽ½ 大éĻĨ +åħħ足 çļĦ +æĹ¢ èĥ½ +åĤ³ çµ± +çijľ ä¼½ +åħ¥ åĽ´ +æħ¢æħ¢ åľ° +æĬ¥ éħ¬ +æī¹ å¤į +å·¥ä¸ļ åĽŃåĮº +ä¸İ åıijå±ķ +èĥ¸ éĥ¨ +åľ¨ ç½ij绾 +åľ¨ç½ij绾 ä¸Ĭ +交 è°Ī +æĽ´ æĶ¹ +åįłæľī çİĩ +ä¸Ŀ绸 ä¹ĭè·¯ +è¡ Ľ +çłĶ åĪ¤ +åĪ ª +åĪª éĻ¤ +è¿Ļ åıª +çļĦ æ°Ķæģ¯ +åĬł å·ŀ +éĴ § +çIJĨäºĭ éķ¿ +ä¸ĸ 家 +æµģè¡Į çļĦ +å¾Ī æľīåı¯èĥ½ +们 éĥ½ +ç»ıèIJ¥ 模å¼ı +è¡Įä¸ļ ä¸Ń +éĢļçŁ¥ 书 +åij½ é¢ĺ +æľ¬ 網ç«Ļ +æ²Ļ çī¹ +åıij åħī +é«ĺ ä»· +å·² çĦ¶ +åıĮ åįģä¸Ģ +ä¸Ĭ è¯ī +ç¿ħ èĨĢ +è¿Ļä¸Ģ å¹´ +大ä¼ļ ä¸Ĭ +éĩ ī +å®Įåħ¨ æĺ¯ +å¾Ĺ 太 +ä¸ĢèĪ¬ 人 +è¿ĺ ç®Ĺ +æĬĺ åıł +æĬķ æľº +çĤ¹ çĩĥ +çİ°éĩij æµģ +åħĶ åŃIJ +ç½ij æł¼ +æİ¥ è¿ĩ +ä¾Ľ è´§ +éĺ´ å½± +åİŁ åħĪ +æį £ +å·¦ 侧 +åħĭ æĭī +æīĵ åį¡ +ç§ij æ¯Ķ +æ±ĩ éĽĨ +åľ°çIJĨ ä½įç½® +è¯Ħ å§Ķ +ç»ĵåIJĪ èµ·æĿ¥ +è¿Ľåħ¥ åĪ° +åı¯ è¡Į +åı¯è¡Į æĢ§ +让 å®ĥ +åĪ¶åº¦ æĶ¹éĿ© +çĶĺèĤĥ çľģ +åĵ Ĺ +åģı åģı +è¡£ çī© +ç¥Ŀ è´º +æºIJ èĩª +并ä¸į 代表 +åĽ½ 度 +好 åĿı +æĿ ĸ +æĿŃ å·ŀå¸Ĥ +湿 度 +é² ¸ +åįļ 彩 +æ³° å±± +æĿij èIJ½ +æĸ° èģŀ +èĤ ĭ +åı¤èĢģ çļĦ +çļĦ ç§ĺå¯Ĩ +ä¸Ģ个 éĹ®é¢ĺ +éģı åĪ¶ +åįĥ 亿 +è¿ĩ 硬 +å°Ħ åĩ» +èĩªçĦ¶ æĺ¯ +产 åĮº +çĤ¹ çĤ¹å¤´ +åı¯ä»¥ 帮åĬ© +说 å®ŀ +说å®ŀ è¯Ŀ +æĪij åıªæĺ¯ +ä¹ĭ ä½Ļ +åIJĮæŶ ä¹Łæĺ¯ +ä¸ŃåĽ½ éĺŁ +建æĪIJ åIJİ +ä¹IJ è§Ĩ +åij¨ å²ģ +èį¯ åºĹ +éĩij åįİ +严éĩį å½±åĵį +è´¨ åľ° +æĹħ éģĬ +åħµ åĻ¨ +æķĻèĤ² æķĻåѦ +离 åİ» +åIJĦå¼ı åIJĦæł· +ä»ĭ ç´ +ä»ĭç´ ¹ +å¼Ģ 头 +å°Ĩ èĩªå·±çļĦ +åIJ¬ åĬĽ +ä¿¡æģ¯ ç³»ç»Ł +ä»İ æł¹æľ¬ +ä»İæł¹æľ¬ ä¸Ĭ +æİĮ 声 +欢 åĸľ +å±ķ åĮº +åķ ¸ +太å¤ļ äºĨ +éĹ² ç½® +èĥ¡ èIJĿåįľ +å§Ķ å®£ä¼ł +å§Ķå®£ä¼ł éĥ¨ +åįĹ éĺ³ +å·ŀ åĮº +ä¸İ æŶ +ä¸İæŶ 俱 +ä¸İæĹ¶ä¿± è¿Ľ +å«Įçĸij 人 +èī¯ å¿ĥ +头 顶 +è´¢ æĬ¥ +ä½Ľ æ³ķ +å¾ µ +åİŁ 件 +åĭ ŀ +çĶ· 篮 +å¤ĸåĽ½ 人 +è¿Ŀ 纪 +æī¾ äºĨ +æįķ æįī +缸 è¯Ĩ +æIJľ éĽĨ +çļĦ ä¼Łå¤§ +ä¸ī ç»´ +å°±è¡Į äºĨ +çĭIJ æľĪ +çĭIJæľĪ å±± +å¸ĮæľĽ éĢļè¿ĩ +èĢĮ 对äºİ +éĿ¢ å°į +åĨĽ åĽ¢ +è¡Ĺ åĮº +æĤ¬ æĮĤ +便 ç§ĺ +æľīä¸Ģ çĤ¹ +ä¼ļè®® ä¸Ĭ +ä¸ĭ æīĭ +廣 åijĬ +äºĶ è¡Į +çŃī åĢĻ +紧紧 åĽ´ç»ķ +æĭ¿ äºĨ +æ¡Į éĿ¢ +ç¥ŀ æĥħ +éĽĦ åİļ +çŀ ³ +楼 ä¸ĭ +å½ ª +äºĭ åıij +åĨį è§ģ +é¤ ĺ +é¢Ħ åĶ® +åİ» çľĭçľĭ +æĪij们 åºĶ该 +ä¸ī 家 +æµ Ĭ +ä¹IJ éĺŁ +çľĭ ä¸įè§ģ +èĦij åŃIJ +æĮģ æľīçļĦ +çĻ½ èıľ +éĹª çĥģ +åĸĿ æ°´ +æݧåĪ¶ ç³»ç»Ł +ä¸ĵ åĮº +æľĿ å»· +æĪij å¿ĥéĩĮ +å±ķ åİħ +èľĺ èĽĽ +åĨ» ç»ĵ +ç² ª +åº IJ +åIJij 社ä¼ļ +åĨ³çŃĸ éĥ¨ç½² +çŁŃ æľŁåĨħ +æĸ° ä¸ļæĢģ +æľ Ķ +æŶ æĬ¥ +使 ä¹ĭ +åĽł åŃIJ +åıĤä¸İ èĢħ +çļĦ 年轻人 +æīĭ 表 +å°ģ éĶģ +为ä»Ģä¹Ī ä¸į +åIJ¸ çĥŁ +æ¯Ĵ ç´ł +åĪij æ³ķ +磫 æŃ£ +身 æĹģ +åİŁ è°ħ +çĽij æĬ¤ +æѤ å¤Ħ +éļ¨ æĻĤ +æŀľ å®ŀ +åĮ»çĸĹ æľįåĬ¡ +ä¸į åIJĪçIJĨ +æIJŀ 好 +çļĦ èĦļæŃ¥ +å¤ĸ å¥Ĺ +ç¶ĵ éģİ +æĶ¾ ç¼ĵ +åģľ çķĻ +æĺŁ çIJĥ +çļĦä¸Ģ éĿ¢ +åĩł ä½ķ +è½® åĽŀ +æ¯Ľ å·¾ +ä¿® çIJĨ +ä¸įçŁ¥ ä¸į +ä¸įçŁ¥ä¸į è§ī +æķ´ 个人 +æ¯ģ çģŃ +åı° å·ŀ +使çĶ¨ 寿åij½ +é»ij çĻ½ +æij¸ ç´¢ +é¼ł æłĩ +éĿ© æĸ° +éº µ +ä¸ĵéŨ 为 +å¾Īå¤ļ æľĭåıĭ +å·¥ä½ľ ç»Ħ +åIJĪ å½± +çĤº ä»Ģ麼 +æŀģ 度 +çļĦ è¿ĽæŃ¥ +å½ĵ ä¹ĭ +å½ĵä¹ĭ æĹł +å½ĵä¹ĭæĹł æĦ§ +è´´ è¿ij +å°º 度 +åľ¨ çİ°åľº +éĻį 临 +åħ»èĢģ éĩij +ç£ ķ +åı¯ä»¥ 使 +管çIJĨ æ°´å¹³ +æľ¬æĬ¥ è®°èĢħ +æ³ķ 令 +åį¡ 车 +举 æµ· +å¤ļ éĩį +åħ¶ éĹ´ +ç´ Ļ +éĩį大 é¡¹çĽ® +æ±Ĺ æ°´ +ç»Ħ å§Ķä¼ļ +ä¿¡æģ¯ åħ¬å¼Ģ +ä¸į论 æĺ¯ +ä¸Ģ åIJ¬ +èĴ¸ æ±½ +æıŃ ç§ĺ +è¶ħ éģİ +触 åıij +å© ¦ +åħ³èģĶ 交æĺĵ +å°± ç»Ļ大家 +好 ä¹ħ +åĢŁ è´· +游æĪı è§Ĵèī² +å¼ĢåIJ¯ äºĨ +æİ ł +åħļçļĦ åįģä¹Ŀ +ä¸ĭ 鼨 +çŁŃ æŶéĹ´åĨħ +å¯ ħ +导 åħ¥ +å·¥ä½ľ ç»ıéªĮ +ä¹Ł åıªèĥ½ +鼷 éľĨ +è·Ł è¿Ľ +åį¡ éĢļ +é¢ĩ æľī +æľº ä½ĵ +æĪĺ士 èģĮä¸ļ +女 主 +ä½ĵåĪ¶ æľºåĪ¶ +足 åįı +èĪĴéĢĤ çļĦ +åĢŁ åı£ +æī¹ åĪ¤ +æķ° åĢ¼ +è« ¾ +éĺ¿æĭī 伯 +åĺ İ +æħ ¶ +è¾¾ 人 +å¼Ģ æ°´ +大 鼨 +温 室 +ä½İ è¿· +ä»į æŧ +éªĹ åŃIJ +亲 å±ŀ +çIJĨ æĻº +æľ¬ åŁºéĩij +å¨ ħ +åĨĻåŃĹ æ¥¼ +å¢Ļ å£ģ +å® µ +èĻ½ çĦ¶æĺ¯ +顺 çĿĢ +åħ« åį¦ +åķĨ çĶ¨ +ä¸į 失 +è¿· èĮ« +顺 便 +æļij æľŁ +欺 è´Ł +é¢ij é¢ij +该 æł¡ +æĸĻ çIJĨ +æ·± æĥħ +åīį éĶĭ +ä¿Ŀ èŃī +èģĮä¸ļ çĶŁæ¶¯ +åħ¬ å¼Ģåıij +åħ¬å¼Ģåıij è¡Į +åħ¥ æĪ· +éł ĵ +åĢ¾ åIJ¬ +éŃ ģ +æĦī æĤ¦ +åĽŀ åIJĪ +åħ¨åĬĽ 以 +åħ¨åĬĽä»¥ èµ´ +åĥ¹ åĢ¼ +èĥ½åĬĽ 强 +ç»ı å¼Ģ +ç»ıå¼Ģ åĮº +è¿ľ æĸ¹ +çļĦ éģĵçIJĨ +缴 åįĩ +缴åįĩ æľº +为主é¢ĺ çļĦ +ç»Ļ æĤ¨ +è¿ĺ æĥ³ +æ¯Ķ æĪij +åĨľ çī§ +æµ· åºķ +çŃ¾è®¢ äºĨ +对äºİ æĪij们 +æŶ 许 +éĶ® çĽĺ +å®ŀéĻħ æݧåĪ¶ +çļĦ æ¨¡æł· +åıįæĺł äºĨ +代 åĬŀ +åĮ» çĶ¨ +éĽĨ ç»ĵ +åıijå±ķ åīįæĻ¯ +æĮĩ çĿĢ +åįİ åĮĹ +è¿Ļ åĩłä¸ª +åIJį æ°Ķ +åĤį æĻļ +èĩª åıij +æ³¢ åħ° +大åĬĽ æİ¨è¿Ľ +èĩª 称 +èįĨ å·ŀ +æIJį 害 +äºĨä¸Ģ åı¥ +æľĢåĪĿ çļĦ +éĩijèŀį åį±æľº +æĢĢ 念 +è¡Į åĭķ +女 æİĴ +ä¸į 解 +ä¼ł éĶĢ +转载 请 +饰 åĵģ +åıª 为 +ä¸İ ä¼Ĺ +ä¸İä¼Ĺ ä¸įåIJĮ +èĥ½ èĢĹ +èı© æıIJ +è¿ij 两年 +è¿Ķ 乡 +马ä¸Ĭ å°± +äºĮ çŃīå¥ĸ +æ°´ 管 +æ³ķ åѦ +çģŃ çģ« +大 å§IJ +åij¨ 转 +æľī æľŁ +æľīæľŁ å¾Ĵ +æľīæľŁå¾Ĵ åĪij +å°į æĸ¹ +ç¥ŀ èī² +æ²¹ èĦĤ +ä¸ī çĤ¹ +ä¸į åĪ©äºİ +äºĭä¸ļ éĥ¨ +å°± è·Ł +å¼Ģ æĶ¯ +å°ı 女åŃ© +åħ±åIJĮ åĬªåĬĽ +çĶļèĩ³ è¿ĺ +è¿Ļ åIJį +è¿Ļ ç¬Ķ +çݯ åį« +æľī ç§į +è§Ĩ åĬĽ +çĨŁ çŁ¥ +åħ¬ç§¯ éĩij +æ¶Īéĺ² å®īåħ¨ +é¢ĩ 为 +大 èħ¿ +éĿ ¶ +çī¹ æķĪ +æľįåĬ¡ åĮº +å¼Ģ åĩº +深度 èŀįåIJĪ +æĹł 忧 +æŁ¥ éĺħ +ç»Ī ç»ĵ +ä¿Ŀ ç¨İ +è¨İ è«ĸ +å½ĵ åģļ +è·³ èĪŀ +å¯ § +女 çİĭ +è®°èĢħ åľ¨ +åħ¨ 产ä¸ļéĵ¾ +è´¯ éĢļ +åħ´ ä¸ļ +éĻį åĪ° +å°ģ éĿ¢ +åħ¨éĿ¢ æİ¨è¿Ľ +奶 èĮ¶ +éĢī åĿĢ +äºĨä¸Ģ åľº +åIJĮ ä¼´ +è®® 论 +æIJ ĵ +诸 èijĽ +诸èijĽ 亮 +å¹² åĺĽ +æµģ æĦŁ +ä¸ĵä¸ļ çŁ¥è¯Ĩ +çĶµ ç«Ļ +åĩı å¼± +åĩº åħ¥ +åIJĦ çľģ +éĿŀ常 é«ĺ +åľ° 毯 +åıij æĸĩ +çĦ ī +çĥ§ çĥ¤ +å£ģ 纸 +æģ¶ åĮĸ +èĬ ¸ +èĥĸ åŃIJ +çĩ Ĵ +çľģ éĴ± +çĻ¾ 强 +çIJĨå·¥ 大åѦ +éĴ¢ æĿIJ +åĽ½æľī èµĦ产 +æĪĺ æľº +æ³Ħ éľ² +åIJİéĿ¢ çļĦ +æ°´ èµĦæºIJ +æ¢ħ èĬ± +åĨĻ çĿĢ +ä¹ĭ 声 +æĹł åı¯ +æĺİ æľĿ +ç«ĭæĸ¹ ç±³ +ç· £ +æĶ¾ è¿ĩ +ç¦ı çĶ° +å¾Ĺ ä½ı +åıĹ ä¼Ĺ +ä¸Ń 级 +çĹħ åıĺ +ä¸Ģ çŀ¬éĹ´ +æĿĥ éĩį +人æĢ§ åĮĸ +åĮ»çĸĹ åį«çĶŁ +ä¸įåĪ° ä½į +æĻºèĥ½ 家å±ħ +饮 çĶ¨ +æ¼Ķ åıĺ +é«ĺ ç´łè´¨ +ä¹Ļ æĸ¹ +åģľ çķĻåľ¨ +èİ· æī¹ +ç©¿ æ¢Ń +客 åľº +æĮ½ åĽŀ +京 åŁİ +çĶŁåij½ åĬĽ +實 éļĽ +çĩ Ī +åĨį çİ° +çİ°å®ŀ ä¸Ń +æľī ä¿¡å¿ĥ +çĸı éĢļ +åĺ´ åĶĩ +鼷 éĶĭ +èıľ åįķ +éħ ¯ +è¶ħ é«ĺ +å¾Ī é«ĺåħ´ +çĶŁ æ®ĸ +éĢł ä»· +误 åĮº +æĨ ĭ +好 æ¶Īæģ¯ +å´ Ń +以 èĩ´ +å¼Ģ çİ©ç¬ij +çĽij è§Ĩ +å·¡ å¯Ł +å¾· å·ŀ +æĹ© æĹ© +éĹª çĶµ +æĪª åĽ¾ +åı¯ä»¥ æł¹æį® +æīĭ èīº +æİ¥ 轨 +ç§į æĹı +æĢĢ éĩĮ +åİ» åĮ»éĻ¢ +ä¸Ģ äºĮ +å¼Ģ éĺĶ +åĩı éĢŁ +ä½Ĩ ä»İ +éĢĻ ä¸Ģ +åĩı åħį +主é¢ĺ æķĻèĤ² +å¼Ģå·¥ 建设 +è¹ ¦ +æľĪ 饼 +ä¸ĭ æ²ī +å°Ĭ 严 +éĻ ĩ +å®ŀ æľ¨ +å»ł åķĨ +声 称 +èĢĥ åľº +å¸ĥ é²ģ +èĩª æĿ¥ +èĩªæĿ¥ æ°´ +éĴ ¾ +å¹´ 以ä¸Ĭ +大 åıĶ +ä»ĸ å·²ç»ı +åħ¨ æĿij +èģĶç³» çĶµè¯Ŀ +为 导åIJij +åĪ¤ å¤Ħ +对 éĺµ +缮 æ¨Ļ +åIJį é¢Ŀ +客 æ°Ķ +横 åIJij +çŃī åĨħ容 +åĩł çĤ¹ +è°Ī 论 +ä¸į ä¹ı +å±ķ çİ°åĩº +è¾ĥ éķ¿ +éĢĨ 转 +å°ı æĻĤ +æĺ¯ å¤ļä¹Ī +æľ¬ æľĪ +è¿ij è§Ĩ +æĪIJç«ĭ 以æĿ¥ +代表 çĿĢ +æĬ¥ å¤į +æĪı æĽ² +è¨Ń åĤĻ +åħ¥ èĤ¡ +å¾ģ æľį +é«ĺ åĩº +èĪŀåı° ä¸Ĭ +å¿ĥ åĬ¨ +两 çĤ¹ +缸 çķ¶ +èĻ Ľ +主 页 +åĩł 家 +æĹł ä¸į +åįı å®ļ +æĸ IJ +å¯ĵ æĦı +åħ¨ 线 +æįķ é±¼ +åı¯ä»¥ ä»İ +æľī è¿Ļæł·çļĦ +æģ¶ éŃĶ +åĮħ åŃIJ +æģ ¤ +å¼Ģå¥ĸ ç»ĵæŀľ +ä¸į æŃ» +èĹ į +弯 æĽ² +æµ· 峡 +éĶĢ æ¯ģ +çļĦ çĭ¬çī¹ +示 æĦı +ä¸įèĥ½ åĨį +èĥ½ æĬĬ +éĺ² çº¿ +ä¸įå°ij äºİ +æ± Ģ +çļĦ éĤ£ä¸Ģ +羣 æĥħ +åŀ ® +被 æīĵ +åĽ½ å®ī +ç¾İ å¦Ļ +è¿Ļ åĩł +åĩº éģĵ +æľįåĬ¡ äºİ +æĪIJæŀľ 转åĮĸ +æīį åįİ +天 é¹ħ +åĩł 个人 +åĢĺ èĭ¥ +èĢ½ 误 +æĬĹ æĪĺ +è¡Į éĬ· +æĿ¥ è¢Ń +åĢŁ éĮ¢ +èįī èİĵ +ä¸¥æł¼ æī§è¡Į +举è¡Į äºĨ +å¤ĸ ç±į +å·² è¾¾ +æĿij åħļæĶ¯éĥ¨ +è¡ Ŀ +éĻį èĩ³ +æµ· éĩı +é¤IJ é¦Ĩ +æĢ¥ å¿Ļ +æ·± è¿ľ +å¾Ģ è¿Ķ +ç¨İåĬ¡ å±Ģ +å¹¿æ³Ľ åºĶçĶ¨ +è®® åijĺ +æĹł æķĮ +çľ¼ åħī +çĥŃè¡Ģ ä¼łå¥ĩ +æŃ IJ +äºĨ äºĽ +è¿Ŀ èĥĮ +è¿Ļ æĺ¯ä¸Ģç§į +ä¸į 稳å®ļ +大家 åĪĨ享 +表 çı¾ +åīį åįģ +è·¯ è¿ĩ +æĴ © +åIJĮ æĥħ +ä¹ł ä¿Ĺ +åıij è´¢ +åºĶ æľīçļĦ +æĿİ æŁIJ +èĤ Ľ +马 åħĭ +éĢļ åijĬ +å·¨ 人 +ä¸Ģ åĽ¢ +éĢĻ 次 +ä¸į äºĨ解 +æĸ½ è¡Į +èij¡èIJĦ çīĻ +åıĺå¾Ĺ æĽ´åĬł +æı £ +åĪĽæĸ° èĥ½åĬĽ +çķħ éĶĢ +表 æī¬ +æ¯Ķ åĪ© +æ¯ĶåĪ© æŶ +åĮ»çĸĹ ä¿ĿéĻ© +æĵį 纵 +伤 亡 +æµİ å®ģ +åıĺ äºĨ +æľ¬æ¬¡ æ´»åĬ¨ +åľŁ 豪 +æĥ³ åĬŀæ³ķ +æĺ ķ +å½ĵ æĻļ +åĩº å±Ģ +çĥŃ è®® +è°Ī è°Ī +æĻĭ åįĩ +åĬ¿ å¿ħ +çĻ» å±± +éĤ£ åĦ¿ +åIJĥ åĪ° +ä¹ĭ åŁİ +å¿« æĿ¥ +æ¹Ľ æ±Ł +第ä¸ī 个 +åħ¨éĿ¢ æıIJåįĩ +å¥ĸ åѦ +å¥ĸåѦ éĩij +æĬķåħ¥ 使çĶ¨ +é½IJ é²ģ +åı¯ä»¥ æĬĬ +åĴĮ ä»ĸçļĦ +è´ŃæĪ¿ èĢħ +æŃ£å¼ı åIJ¯åĬ¨ +åįİ æ¶¦ +ä¸įæĸŃ å®ĮåĸĦ +éĴ¢ æĿ¿ +ç´¯ 积 +满 èĦ¸ +åĽĽ æĸ¹ +è´¢ çī© +ä»ĸ们 ä¼ļ +å¤ı æĹ¥ +éĤ£ 个人 +éĿł çĿĢ +çĤ¹ äºĨ +çĤ¹äºĨ çĤ¹å¤´ +æ© ĭ +åıĪ 好 +åıĪ好 åıĪ +åıĪ好åıĪ å¿« +éĺµ éĺµ +å°ģ 建 +æľ¬ çĶ° +çī©ä¸ļ æľįåĬ¡ +èĩªè´¸ åĮº +åIJ ı +便åĪ© åºĹ +åĽ½å®¶ æłĩåĩĨ +éĿ¢ ç²ī +èī° è¾Ľ +æĶ» åħ³ +æīĵ åĮħ +车 éĺŁ +人 éĢī +åı¯ ä¸įæĺ¯ +äºĮ åįģå¹´ +åIJį å¸Ī +浦 举 +åħ¬ è¯ģ +è¿IJ éĢģ +æĺ¯ æľĢ好çļĦ +æŁĶ åĴĮ +çİĭ æŁIJ +çĹħ æĪ¿ +åĨ¶ éĩij +ä¸Ģ件 äºĭæĥħ +åį ¤ +åı¯ æݧ +çī Ł +æĭ Ĥ +å·² äºİ +人 éĢł +çĶŁçī© åĮ»èį¯ +ä½ĵ çİ°åĩº +èĤ² åĦ¿ +èĢģ å®ŀ +åľĸ çīĩ +è« ¸ +ç´¯ äºĨ +æĦŁåħ´è¶£ çļĦ +åĽ¾çīĩ æĿ¥æºIJ +ä¹Ł æĺ¯ä¸Ģç§į +æ¾İæ¹ĥ æĸ°éĹ» +æŶ 表示 +åħī è¾ī +æĬ¥ åºŁ +å²ģ æŶ +éħ ® +æ£Ģ ä¿® +åıĺ éĢŁ +åıĺéĢŁ ç®± +åľ¨ èģĮ +éı ¡ +æį Ĥ +çĿ£ åĬŀ +æ°¸ ä¸į +åģļ ä¸ĢäºĽ +åİĨ æŶ +å·¥ç¨ĭ æľºæ¢° +æģ° å½ĵ +å°± åľ¨äºİ +称 åij¼ +éĢļ常 æĺ¯ +æł· å¼ı +åij¨ ä¸Ģ +èĭ± éķij +åĿĩ 线 +ä¼ł éĹ» +çĶ¨æĪ· ä½ĵéªĮ +èµŀ åIJĮ +骨 æĬĺ +为主 ä½ĵ +æ±Ł å±± +æ¸ħ æľĿ +æĶĢ åįĩ +ä¸į çĽ¸ä¿¡ +éĿ ´ +æѦ åĬŁ +åĭ¤ åĬ³ +æĿ¥ æī¾ +å°Ĩ æĮģç»Ń +丫 头 +æ¨Ļ æºĸ +è£ ´ +深深 çļĦ +åŃķ èĤ² +è§ĦåĪĴ 建设 +æ¸ħ çĪ½ +ç²¾åĩĨ æī¶è´« +æīĵçł´ äºĨ +è¿Ļä¸Ģ 天 +å·¥ä½ľ æĢ»ç»ĵ +æĹħ ç¨ĭ +举 èIJ¥ +æĶ¾ å°Ħ +æľī åĩłä¸ª +éĿŀ çī©è´¨ +åIJĥ å¾Ĺ +åĹ ¨ +ä¼ļ åıijçĶŁ +篮 æĿ¿ +å¼Ģ å°ģ +麻 å°Ĩ +èıı æ³½ +ä¸į åIJĪ +ç³»åĪĹ äº§åĵģ +èѬ å¦Ĥ +ç¾İ èªī +èĩªå·± åĸľæ¬¢ +交æĺĵ ä¸Ńå¿ĥ +åIJĪ åĶ± +使 æĪij +åĥı ç´ł +带 éĺŁ +ä½Ĩ 对äºİ +æĬĬ è¿Ļ个 +èĤĿ èĦı +åįķ纯 çļĦ +æĶ»åĿļ æĪĺ +缼 ä¼ļ +åijµ æĬ¤ +æª Ģ +赶 ä¸Ĭ +æ¥ Ĭ +ä¹ħ äºĨ +ç¡ Ŀ +çŃĶ é¢ĺ +ä¿ĿæĮģ çĿĢ +è§ģ è¯Ĩ +çĤ¹ åĦ¿ +åįĬ 个æľĪ +æ» ĩ +浸 泡 +ä¼ł éĢģ +åľ¨ å¸Ĥåľºä¸Ĭ +ä¹ĭ 乡 +çī¹ éķ¿ +éĽ ŀ +èª ł +身 å¤Ħ +æŁł 檬 +身 ç©¿ +çľģ åħ¬å®ī +çľģåħ¬å®ī åİħ +åıĻ åĪ©äºļ +åĩł åĪĨéĴŁ +人 åĢij +åľ° 段 +èĩª åѦ +ä¹Ł è¶ĬæĿ¥è¶Ĭ +èģĮ æĿĥ +æĸ § +èĩ » +å½Ĵ 纳 +驾 é©Ń +éĥ¨åĪĨ åľ°åĮº +没æľī æĥ³åĪ° +æĴ ĩ +ä¹Į é²ģ +ä¹Įé²ģ æľ¨ +ä¹Įé²ģæľ¨ é½IJ +èĤ² 人 +çļĦ æŃ¥ä¼IJ +延 æľŁ +æ²¹ æ°Ķ +åģļ å®Į +åľ£ åľ° +丰 åİļ +宽 带 +åı¯éĿł çļĦ +åºŃ éĻ¢ +åŃ ľ +å°ı康 社ä¼ļ +å®īåħ¨ 管çIJĨ +å¹´ 第 +æİĴ 污 +èĥĮ åĮħ +家 ä½ı +åħ¶å®ŀ å°±æĺ¯ +ä¼ļ è§ģ +帮åĬ© ä¼ģä¸ļ +ç½ij è´Ń +æĺ¯ ä¸įä¼ļ +飯 åºĹ +æŃ» åİ» +åħįçĸ« åĬĽ +æľ ķ +åĸĿ äºĨ +è½» å¾® +个æľĪ åĨħ +ç»Ħ åĽ¢ +åĴĮ å®ĮåĸĦ +é¸ ½ +æıIJ éĢŁ +西å®ī å¸Ĥ +ä¸Ńå¿ĥ 主任 +æŶéĹ´ 为 +æľŁ æĿĥ +è¶ ķ +ä¸įä»ħ è¦ģ +æľį ä»İ +é¡ĺ æĦı +ä¸į å°ı +ä¸įå°ı çļĦ +ç° ĩ +çª ¦ +åĪĩ æĪIJ +åĵĪ åĪ© +天 羣 +ä¸Ģ次 次 +éĩij å¸ģ +æĢİä¹Ī èĥ½ +ç½ij è´· +ä¼ļ计 å¸Ī +çŁŃ 缺 +对 æłĩ +åıĺå¾Ĺ æĽ´ +åīį åĩłå¤© +éĺ² æ±Ľ +彩 èĻ¹ +åĵģ ä½į +表 æł¼ +严 å¯Ĩ +æ¯Ľ åĪ©çİĩ +çļĦ åį±å®³ +å½ķ åĪ¶ +æ°´ åĬ¡ +èĥ½å¤Ł 让 +å¹³ æĿ¿ +ä¹³ æĪ¿ +è¸ı å®ŀ +é¦ĸ åĪĽ +é¦Ļ èķī +æĬ¥ 表 +ä¸Ģ æĬ¹ +åĩºçĶŁ äºİ +è²» çĶ¨ +åĩº 让 +åIJĪæ³ķ æĢ§ +å°¼ åħĭ +åĨ° åĨ· +é¦Ļ æ°Ķ +åı· 称 +èµ· çłģ +åŁİ åİ¿ +çİ© èĢį +ä¸Ĭ éĻIJ +ä¼ļè®® ç²¾ç¥ŀ +æĹģè¾¹ çļĦ +便 ä¼ļ +æıŃ æĻĵ +çİ© æĦı +éĽª å±± +åIJij çĿĢ +ä½ĵèĤ² åľ¨çº¿ +说æĺİ ä¹¦ +åĮĸ èĤ¥ +åħļç»Ħ 书记 +åĬ¨ 人 +ä¹ĭ æīĢ +æľĪ èĩ³ +æľĢå¿« çļĦ +èĬĤ åģĩæĹ¥ +ä¸ĵ åľº +èĢĥ ä¸Ĭ +çª Ł +é²ľ è¡Ģ +è¾ĥ强 çļĦ +æĤĦ çĦ¶ +å¤ļ个 åĽ½å®¶ +çªĹ å¸ĺ +æŀģ å¤§åľ° +ä¸įçĶ¨ æĭħå¿ĥ +è¿Ļä¹Ī åģļ +åĥ¹ æł¼ +ç¾İ丽 乡æĿij +å°ıæŶ åĨħ +ç´§ è¿« +大 çģ« +èĥ³ èĨĬ +æĵįä½ľ ç³»ç»Ł +æ®ĭ çķĻ +åĨĻ åĩº +ç¦ģ å¿Į +åĬłçĽŁ åºĹ +è¿ij çĻ¾ +便 åı¯ +æķ´æĶ¹ æݪæĸ½ +éĩĩ访 æŶ +åĶIJ 代 +æ·±åĮĸ æĶ¹éĿ© +çŁ ¢ +éĥ½ åĸľæ¬¢ +è¶ĬæĿ¥è¶Ĭ é«ĺ +èĬ± æľµ +头 çĸ¼ +å®ī 康 +å¢ŀéķ¿ çİĩ +çľ¼ çľĭ +å°±æĺ¯ ä¸ºäºĨ +èĢĮ 导èĩ´ +åĬłå¿« 建设 +èĬ± æł· +åĨħå¿ĥ çļĦ +æĺĨ å±± +è³ĩ æºIJ +åĽŀåĪ° 家 +èıĬ èĬ± +æ°´ éĩı +å¾ģ ä¿¡ +è¡ĮæĶ¿ åĮº +ä¹ĥ æĺ¯ +æĬķèµĦ é¡¹çĽ® +å«ģ ç»Ļ +ç¥ŀ åľ£ +ç¨ ł +æľ¬æĿ¥ å°± +éĢIJ ä¸Ģ +èģĮä¸ļ æĬĢæľ¯ +ä¸įèī¯ ä¿¡æģ¯ +æīĺ è¿IJ +åIJ¯ 示 +ä¹ĭ åħ§å®¹ +éŁ ¶ +奢 åįİ +æıŃ ç¤º +æĪIJ为 ä¸ŃåĽ½ +æ¶Īè´¹ åĵģ +åħ¬ çĶ¨ +æIJŀ å®ļ +请 ä½ł +æŁ ļ +åĨħ è¡£ +ä½Ĩ ä»ĸ们 +ä¿Ŀ 湿 +该 åİ¿ +饱 åĴĮ +æݨ åIJij +èµĦæĸĻ æĺ¾ç¤º +ä¸į å½±åĵį +人 人éĥ½ +åıijå±ķ 壮大 +åħ»èĢģ æľįåĬ¡ +çĶŁæ´» æ°´å¹³ +åIJĦ åİ¿ +ä½ł éľĢè¦ģ +说 çļĦæĺ¯ +å¤ĸ åªĴ +æѤ 人 +次 è¦ģ +追 赶 +åºĶ该 å¦Ĥä½ķ +æĹ¥ åĩĮæĻ¨ +çķ¥ æľī +éĥ½ æĥ³ +游 ä¹IJ +è¿Ļ款 游æĪı +å¹³ æ·¡ +æĺ¯ä¸Ģ åĢĭ +å¤ĩ èĢĥ +åĪ¶ æŃ¢ +ä¸Ģå®ļ èĥ½ +å¾Ĵ å¼Ł +以 çĤº +åįĥ åħĥ +äºĶ åħŃ +迪 士 +迪士 å°¼ +éĺ³ æĢ§ +åĨ¬å¥¥ ä¼ļ +å°±æĺ¯ åĽłä¸º +æĮĤ éĴ© +æ¦Ĥ åĨµ +åıªè¦ģ æľī +æ²¹ çĶ» +åľ° æłĩ +ä¸Ĭ è°ĥ +产ä¸ļ åĽŃåĮº +åħ« åįģ +æ£ ± +液 æĻ¶ +æĿij å§Ķä¼ļ +çŃ¾çº¦ 仪å¼ı +è¿Ļ åħ¶ä¸Ń +åĨĻ éģĵ +示èĮĥ åŁºåľ° +éĩİçĶŁ åĬ¨çī© +鼻åŃIJ ä¿¡ç®± +åĽ½éĻħ è´¸æĺĵ +人 æĿĥ +ä¿Ŀ 管 +èĭ¥ æĤ¨ +åİĭ æĬij +é» Ľ +åľ° çľĭçĿĢ +éĻ ° +ä¸Ģå¹´ å¤ļ +ä»İ 容 +ä¸Ń æĸŃ +å¯Ł è§ī +移 交 +éĶ ¯ +æĪĸ许 æĺ¯ +ç¶ ł +两 项 +æľĢ åĸľæ¬¢ +æľĢåĸľæ¬¢ çļĦ +å¤ľ éĩĮ +åIJĮ ä»ģ +åĪĽæĸ° 驱åĬ¨ +è°ģ èĥ½ +é£ ¾ +åħī åѦ +åİ Ħ +èĦ± é¢ĸ +èĦ±é¢ĸ èĢĮåĩº +è¿ ¦ +æĺ¯ ä¸įåı¯èĥ½ +çª ¥ +èĥ½ 满足 +宽 度 +伦 çIJĨ +åı¯ä»¥ èİ·å¾Ĺ +转 ä¼ļ +å±± æĿij +éĵº 设 +åĩº åĩ» +æĸĩåĮĸ èīºæľ¯ +ä¼ļè®® 室 +æŃĮ 声 +æ» Ķ +èIJİ ç¼© +æľįåĬ¡ åijĺ +åıij表 äºĨ +æĸ¼ æĺ¯ +æĺİç¡® è§Ħå®ļ +ç»´ å¥ĩ +æ°´ 产 +æĬķ ä¿Ŀ +éĺ´ éģĵ +赶 å¿« +夺 å¾Ĺ +ä¸ĭ åįķ +çī©æµģ åħ¬åı¸ +çݯ ç»ķ +å½ Ī +ä½ľé£İ 建设 +æĹħ游 æĻ¯åĮº +æľī æĽ´å¤ļçļĦ +丰å¯Į å¤ļ彩 +çIJĨè´¢ 产åĵģ +åĩº å·® +ä»İ严 æ²» +ä»İ严治 åħļ +缸 å¹² +æ»ĭ 润 +主åĬŀ æĸ¹ +åī§ åľº +æ»ļ çIJĥ +æ©Ħ æ¦Ħ +èĩªä¸» åĪĽæĸ° +éĢļ å¾Ģ +æł¼ å°Ķ +çļĦ ä¼ĺçĤ¹ +èĥĮ ä¸Ĭ +çª ľ +çĪĨ åĩº +å¹³ æķ´ +ä¸Ģ èĦļ +åħ¨ä½ĵ åijĺå·¥ +éĻIJ å®ļ +åŁİéķĩ åĮĸ +æ· ³ +éĢ® æįķ +è¡ĮåĬ¨ 计åĪĴ +æīĵ å¾Ĺ +åİļ éĩį +纪å½ķ çīĩ +åĿļ ä¿¡ +央 ä¼ģ +åĨį ä¹Łä¸į +天 涯 +åıĤèĢĥ èµĦæĸĻ +æľī æ¯Ĵ +åIJ¸ 纳 +è¶Ĭ åıij +éĩįè¦ģ æĦıä¹ī +åĽ½éĺ² éĥ¨ +è¿Ļ个 è¡Įä¸ļ +æĻ® æŁ¥ +å¼Ĥ æĢ§ +延 è¿Ł +å°ı å¹ħ +èī² æĥħ +综åIJĪ æ²»çIJĨ +æŃ£æĺ¯ åĽłä¸º +产ä¸ļ ç»ĵæŀĦ +çłĶ究 æĬ¥åijĬ +åģľ ä¸ĭ +éķ¿ èĢģ +éĩĿ å°į +åįĹ京 å¸Ĥ +çģĮ æºī +转 è¿IJ +欺 è¯Ī +éĢł åģĩ +åĪĨå¸ĥ å¼ı +æĦŁ 触 +æĪij å½ĵæŶ +åıij è§ī +åĽ¾ 纸 +æĶ¹ èī¯ +çĭł çĭł +åĨ² åĪº +æĸ° 京 +æĸ°äº¬ æĬ¥ +ç¥ŀ åĻ¨ +秸 ç§Ĩ +çĪ º +å°Ĩ è¿İæĿ¥ +å·¥ ä¿¡ +工信 éĥ¨ +éĻIJ éĩı +æŃ¢ æįŁ +åѦä¼ļ äºĨ +åįİ çĽĽ +åįİ缼 é¡¿ +å¾Į ä¾Ĩ +ä¸ĭéĿ¢ æĺ¯ +ä¸ĭéĿ¢æĺ¯ å°ı +æIJ¬ è¿IJ +ç¾İæľ¯ é¦Ĩ +æ¸ħ åĩī +å¤ļå¹´ åīį +è© ŀ +åįĥ ç±³ +表 è¿° +æ±Ł éŨ +åĬłæ²¹ ç«Ļ +æľ¬ èĥ½ +导 读 +åĽ´ è§Ĥ +并 åIJij +åŁºæľ¬ æĥħåĨµ +æīĵ å¼ĢäºĨ +è¿Ļ ä¸ī个 +æ±ķ 头 +强 æľīåĬĽ +强æľīåĬĽ çļĦ +è¿Ľ åľº +ä¹Ŀ æ±Ł +çIJĥ æĺŁ +好çľĭ çļĦ +大 æĪ· +æ¹ ¯ +å¥ĩ å¦Ļ +ä¹IJ åĻ¨ +æĪijçļĦ å¿ĥ +çľī 头 +åĨľä¸ļ çĶŁäº§ +ç¼ĸ çłģ +åŁº ç¤ +åŁºç¤ İ +天 æĸĩ +åĢĭ人 è³ĩè¨Ĭ +åİ» è¿ĩ +èģĨ åIJ¬ +æĶ¾ åģĩ +ä¸į åħ·å¤ĩ +æ·Ģ ç²ī +大 佬 +åħ¨ 天 +åħ¨éĿ¢ 建æĪIJ +éļIJ å½¢ +ç¼ħ çĶ¸ +åIJ ³ +è¡ĮæĶ¿ æī§æ³ķ +åŁİ åł¡ +èİ« æĸ¯ +èİ«æĸ¯ ç§ij +æīĢæľī æĿĥ +éĽĨ åľĺ +å±Ģ åī¯å±Ģéķ¿ +åĩłä¹İ 没æľī +æ´ģ åĩĢ +çĶµå½± èĬĤ +åŃ© ç«¥ +æīĢ åģļçļĦ +æ¸ħ 代 +æĸ° çīĪ +éĵĿ åIJĪéĩij +为 æĬĵ +为æĬĵ æīĭ +åĪ¤ å®ļ +çī¹ 产 +æīĭ æ©Ł +ä¸įåı¯ æĪĸ +ä¸įåı¯æĪĸ 缺 +å¸Ĥåľº è§Ħ模 +åĿ ¯ +åĮ» åѦéĻ¢ +å¿« è¦ģ +èĮ ľ +æĬĺ èħ¾ +äºĨ è¿ĩæĿ¥ +æĬ¥åijĬ æľŁåĨħ +çī© ç§į +ç»Łè®¡ å±Ģ +æī© 建 +æ¶ ħ +责任 人 +éĺ İ +è¯Ħ è®® +å¾Ģ äºĭ +æīĢ 示 +æķ´ æ´ģ +éĹº èľľ +æĹħ éĢĶ +å®ŀ è®Ń +ä¹ĭ 称 +å·´ 士 +éĢŁåº¦ å¿« +ä¸įä»ħ å¦ĤæѤ +å®Ŀè´µ çļĦ +åºŁ çī© +æ²³ æ°´ +æİ¥ 纳 +ç²¾ æ¹Ľ +åħ¶æ¬¡ æĺ¯ +顺 å¾· +åħ¬åħ± åį«çĶŁ +è¤IJ èī² +ä¸į æĥľ +æĬĢæľ¯ æľįåĬ¡ +æİ · +æ±Ĥ èģĮ +ä¸ī 峡 +æĬķåħ¥ åĪ° +太 åIJİ +åIJ¯åĬ¨ 仪å¼ı +缴æİ¥ å½±åĵį +æĸ° 款 +个 乡éķĩ +çĻ¾ 亿 +åº « +ä¹Ł æŃ£æĺ¯ +åı¶ çīĩ +æľĢæĹ© çļĦ +æĪĺ 绩 +å·¥ æľŁ +æĻļ æľŁ +è¿Ļæł· 说 +è¯į è¯Ń +ä¾ Ħ +æķ£ çĥŃ +éĽĨæĪIJ çĶµè·¯ +åIJį è¯į +æĻº åķĨ +æĭ¥ åłµ +çĭĤ 欢 +è¿Ļ èĪ¬ +æµ´ 室 +åijķ åIJIJ +æľªæĿ¥ åıijå±ķ +ä¸īä½į ä¸Ģä½ĵ +åªĴ é«Ķ +ä¸įå¾Ĺ 转载 +åĽłä¸º 她 +æĺ¾ç¤º å±ı +ä¾Ľ æļĸ +éĨ« éĻ¢ +æľī æĦıæĢĿ +æľīæĦıæĢĿ çļĦ +娱ä¹IJ åŁİ +åįµ å·¢ +åĪĽéĢł åĬĽ +ç«ł èĬĤ +人大 常å§Ķ +èĢĮ çİ°åľ¨ +å¤ĸ å©Ĩ +å¢ŀ æĮģ +äºĶ åįĥ +èĢģå¸Ī 们 +æ´Ľ æĿī +æ´ĽæĿī 磶 +æİĮæı¡ äºĨ +ä¸ŃåĽ½ æĸĩåĮĸ +æĸ° æĶ¿ +主è¦ģ çĶ¨äºİ +åıij çĥ§ +类似 äºİ +åĮĹ æŀģ +æĪij们 认为 +å¼¥ 漫 +åħ¨çIJĥ ç»ıæµİ +é¢ IJ +ä¸Ģèµ· è£ħä¿® +æĶ Ĵ +æĭī èIJ¨ +帶 ä¾Ĩ +åĨ· æ°´ +ä¸ī åĨľ +æĿ¿ æĿIJ +è¿ŀ è¿ŀ +éĵ ® +ç»ıèIJ¥ çIJĨ念 +å±± 顶 +å¾Ī æĥ³ +çĺ « +å§ĭç»Ī ä¿ĿæĮģ +åľ¨ 广å·ŀ +ä¸įåIJĮ æĦı +åıĺ åİĭ +åıĺåİĭ åĻ¨ +产 éĶĢ +表 éĿ¢ä¸Ĭ +æīĢ以 ä»ĸ +ç»ıéªĮ 丰å¯Į +éĥ¨ å§Ķ +åħµ åĽ¢ +æīĢ è¿° +æķ¦ çħĮ +ç»ıèIJ¥ èĮĥåĽ´ +åı£ è¯Ń +失 ä¿¡ +æ¯ı个人 çļĦ +æīĭ æĮģ +æģIJ æħĮ +åł¡ åŀĴ +é¦ ħ +éĵ¸ éĢł +æĭ¿ åĩºæĿ¥ +æİ¢ æµĭ +大家 ä¸Ģèµ· +å¥ § +å®ŀè´¨ æĢ§ +å°ı åĦ¿ +èĩº åįĹ +èĩºåįĹ å¸Ĥ +å¼Ģåıij èĢħ +åı¯ æł¹æį® +ç®± åŃIJ +饺 åŃIJ +å¿Ļ çĿĢ +æĿ¥ ä¸įåıĬ +缸 ä¼ł +åĽ½ ç½ij +èħ¹ æ³» +è¿ĻéĩĮ æľī +é£İ æĻ¯åĮº +åıĤ ä¿Ŀ +æŃ» èĢħ +æĪ´ ä¸Ĭ +æ©Ł æ§ĭ +è¯ķéªĮ åĮº +ä¼ł æİĪ +æµ· è¾¹ +泪 æ°´ +缸åħ³ åĨħ容 +éĥij å·ŀå¸Ĥ +åħij çİ° +两 åij¨ +èĬľ æ¹ĸ +çĶµåŃIJ ä¿¡æģ¯ +红 å¤ĸ +æĹħ游 å±Ģ +å¾Ģå¾Ģ ä¼ļ +è¿ħ çĮĽ +ä¼ł 羣 +æ¸ħ æ¾Ī +å°± è¿ij +微信 群 +ç³»åĪĹ æ´»åĬ¨ +ç»ı常 ä¼ļ +è§Ĥ æµĭ +å¿ĥå¾Ĺ ä½ĵä¼ļ +éĻĪ åĪĹ +åĮĹ æĸĹ +è« ® +è«® è©¢ +è¿ĺæĺ¯ ä¼ļ +æµĭ ç®Ĺ +æĺŁ ç©º +宽 容 +çī©ä¸ļ åħ¬åı¸ +æĪĴ æĮĩ +å¸ħ æ°Ķ +ä¸ĢæŃ¥ æŃ¥ +åħ± 鸣 +åĨ³ ä¸į +æİ¥ 管 +å¦ĩ èģĶ +æ¯Ķ åĸ» +é²ģ è¿ħ +æĮģ çºĮ +缸 亲 +å¨ģå°¼æĸ¯ 人 +ç«ĭ 项 +åĪ Ŀå§ĭ +èĩª åĪ¶ +è¿Ī è¿Ľ +ä¸Ĭ æ±½ +å®ı ä¼Ł +æł¹æľ¬ 没æľī +æĸ°åĨł çĹħæ¯Ĵ +åĵª ç§į +康 åħ» +è¡° èĢģ +å½ķ åĥı +é«Ķ é©Ĺ +ç»ij å®ļ +é¢Ŀ 头 +äºĶ æľĪ +èĬ± å¼Ģ +ä¸Ģ线 åŁİå¸Ĥ +åĪ° åľº +æĬķ éĻį +çĹĺ çĹĺ +åıĹ ä¸įäºĨ +æīİ æł¹ +æĽ´ ä½ķåĨµ +æĬ½ æŁ¥ +åĩº è·¯ +审议 éĢļè¿ĩ +ä¸į åĥħ +èī² è°ĥ +çĻ¾ ä½Ļ +èĤł éģĵ +æ·±åİļ çļĦ +马 åĬĽ +æĹ© æĻļ +æŃĮ èĪŀ +éĺ² æĻĴ +æľĢåIJİ ä¸Ģ个 +樱 èĬ± +å°ıä¼Ļ åŃIJ +åľ¨ å½ĵåľ° +å°ıä¼Ļä¼´ 们 +èµ· æºIJ +åħ¨ åªĴä½ĵ +ç° ½ +éħ± æ²¹ +æĹłè®º å¦Ĥä½ķ +裤 åŃIJ +åģľ 产 +ä¸įçĶ± å¾Ĺ +çīµ å¼ķ +ä¼ł åĬ¨ +ä¹Ŀ é¾Ļ +åĬł åĽº +ä¹Łä¸į æķ¢ +æĬĢæľ¯ æĶ¯æĮģ +ä¸Ĭ å²Ĺ +ç»ıéªĮ åĴĮ +æł¼ æŀĹ +åIJ¸ éĻĦ +æľªæĪIJ å¹´ +奢ä¾Ī åĵģ +追 æį§ +好 ä¸į容æĺĵ +èķ´ åIJ« +ä¿Ŀ å®ļ +æĬ¥ ä¸ļ +æµ· åĨħå¤ĸ +ä½ł çİ°åľ¨ +æ²¹ èĢĹ +è´¨éĩı 管çIJĨ +æ½ľ æ°´ +丽 æ±Ł +转 åħ¥ +è¿Ļä¹Ī ä¹ħ +æĺİ ä»£ +责任 åĪ¶ +éĩį å·¥ +大 å·´ +触 åıĬ +èµ· åĪĿ +大 å¦Ī +æĸ¯ å¡Ķ +åĨĽ å·¥ +书 éĻ¢ +å³ ¨ +æݨ çIJĨ +è¿Ļç¯ĩ æĸĩç«ł +è¿ģ 移 +åľ¨ åIJĮä¸Ģ +ç»Ĩ ç»Ĩ +åīĬ å¼± +书 æĪ¿ +ç¶ĵ 常 +è¯ķ é¢ĺ +æĤ£ ä¸Ĭ +çĻ«çĹ« çĹħ +åĨ² æ´Ĺ +å¤ĸ æı´ +åħĭ åĪ¶ +åįģ æľĪ +åģļ ä¸įåĪ° +ç¾İ åĮĸ +å¦Ĥ æľŁ +è¿ĺ éľĢ +天 åºľ +å°± æĦıåij³çĿĢ +çļĦç¡® æĺ¯ +éªĹ å±Ģ +å°ıç»Ħ èµĽ +è© © +ä¹Ŀ å¹´ +æĻĵ å¾Ĺ +çłĶ究 人åijĺ +大 éħĴåºĹ +ç§ij åѸ +åħŃ åIJĪ +çķĮ å®ļ +车 è½½ +å¼Ģ çĿĢ +毫 æĹłçĸij +毫æĹłçĸij éĹ® +è¿IJ ç»´ +ç¦ģ åĮº +èĦ± èIJ½ +讲 å¸Ī +产ä¸ļ åŁºåľ° +é«ĺ æĢ§èĥ½ +åħī 彩 +çİ° éĺ¶æ®µ +åĩ ¿ +è¾ĥ å·® +饮 çĶ¨æ°´ +éĸĭ çĻ¼ +ç½ij åIJ§ +çĮ´ åŃIJ +æѦ æŀĹ +å®ī åİ¿ +ä¸įåı¯ æĢĿ +ä¸įåı¯æĢĿ è®® +éĬ· åĶ® +è´« ç©· +为 åķ¥ +éº ĵ +å¹¾ åĢĭ +è§Ħ模 以ä¸Ĭ +æı ļ +被 åĽ° +缺 å¸Ń +å¿« é¤IJ +æĬ¢ åįł +æĻ Ł +å¤į æ´» +æľ¬æĬ¥ 讯 +åĪĽ ä¸ĭ +æµ· 滩 +éĩı 产 +å¦Ĥä½ķ åİ» +车 ä½į +å¯ ĩ +äºĮ åįģåĽĽ +ç»ıæµİ æįŁå¤± +éħįå¥Ĺ 设æĸ½ +åŁºæľ¬ éĿ¢ +äºī 论 +就好 åĥı +çłĶ究 æĪIJæŀľ +éĻĪ è¿° +æīĵ åĬ¨ +ä¸ĭ å·´ +ç§Ĵ éĴŁ +对 人ä½ĵ +æĬĢæľ¯ çłĶåıij +åİŁ åŃIJ +æĺ¯ä¸Ģ 项 +äºĨä¸Ģ 份 +æĮĩ çĶ² +çĶ¨ éĩı +è¿ĺä¸į å¤Ł +æĶ¿åºľ éĩĩè´Ń +çŁ¥è¯Ĩ çĤ¹ +ä¸ŃåĽ½ 梦 +å¾Ī å¼Ģå¿ĥ +礼 è²Į +éĿŀ常 å¤ļ +éĿŀ常å¤ļ çļĦ +åĽ ļ +æĹħ é¦Ĩ +å°½ æĥħ +æŃĮ åĶ± +æ²Ļ é¾Ļ +车 åİ¢ +客 æµģ +åģı å·® +积累 äºĨ +æ¡ Ķ +çĶ» çĶ» +ä¹Ł åºĶ该 +åºĶçĶ¨ ç¨ĭåºı +èĥĥ èĤł +以 å¾Į +豪 å®ħ +æ·± åĬłå·¥ +缴 è¨Ģ +åĮĸ çŁ³ +åĽ½ éģĵ +ä¸ĥ 个 +ä»İèĢĮ 使 +èĤł èĥĥ +æĹ¥ è¶ĭ +çĪ¶ åŃIJ +ç· © +æĭĽ çīĮ +产 å¦ĩ +çķª èĮĦ +æĪij éĻ¢ +建çŃij å·¥ç¨ĭ +å±ķè§Ī ä¼ļ +家éķ¿ 们 +åĨľ ä½ľçī© +æĹ¥ å¤ľ +æĶ» æĵĬ +è§Ħ éģ¿ +èĪŁ å±± +便 æ°ij +åħ« åŃĹ +ä¸į æĽ¾ +æĶ¯ éħį +çĨ¬ å¤ľ +人 é¡ŀ +ç´Ģ éĮĦ +ç»ıèIJ¥ æ´»åĬ¨ +大 涨 +å¸Ĥå§Ķ 常å§Ķ +åĪĨ éIJĺ +ä¸Ģ个 èģĮä¸ļ +çĹħ åĽł +è¿Ļ 对äºİ +ä¸įå¾Ĺä¸į 说 +åıijçĶµ æľº +æľīæīĢ 帮åĬ© +缮æłĩ ä»»åĬ¡ +åĽł åľ° +åĽłåľ° åĪ¶ +åĽłåľ°åĪ¶ å®ľ +å°Ĩ è¾¾åĪ° +ç²Ĺ ç³Ļ +稳 åĽº +å« £ +çİ°åľ¨ å¾Īå¤ļ +ä¸ĸçķĮ 级 +å¼ł æŁIJ +çĤ¹ ç¼Ģ +èij µ +社ä¼ļ ç»Ħç»ĩ +å¾Ģ åIJİ +åĬł æģ¯ +åĻª 声 +æľī åħ´è¶£ +为æĤ¨ æıIJä¾Ľ +æ²¹ æ¼Ĩ +ç¬¬åĽĽ å±Ĭ +çļĩ 宫 +ä¹Ĵ ä¹ĵ +ä¹Ĵä¹ĵ çIJĥ +éļ¨ èijĹ +éģ© åIJĪ +åįĹ éĿŀ +æĵ ´ +西 æ´ĭ +åĬł å¯Ĩ +æĪIJåĬŁ 举åĬŀ +åı£ æ°´ +æĪIJ 年人 +æīĢ æıIJä¾ĽçļĦ +éļĶ å£ģ +åľ¨ 京 +å½ĵåľ° æŶéĹ´ +çŃī åIJĦç§į +é£İ æ°Ķ +å±ĭ éĩĮ +ä¸Ģ åŃĹ +çļĦæŶéĹ´ éĩĮ +åĺ¿ åĺ¿ +å¿« 讯 +ä¸Ń åľº +ä¸Ģ çĵ¶ +æ» ķ +é¢Ĩ è·ij +好 èݱ +好èݱ åĿŀ +没 åħ³ç³» +åĩº å¢ĥ +ä¸įæĺ¯ ä¸Ģ个 +éĥ½æĺ¯ éĿŀ常 +éľĩ åĬ¨ +èİ· èĥľ +åįļ å¼Ī +æĬļ åħ» +对 ç«ĭ +æľįåĬ¡ æľºæŀĦ +è°£ è¨Ģ +社ä¼ļ ç§ijåѦ +åIJ¬è¯´ è¿ĩ +æī ³ +æīĵ 磨 +åı£ æľį +好 åĥıæĺ¯ +以åıĬ åħ¶ä»ĸ +çī¹ è´¨ +亲 è¿ij +ä¸Ģ ç»ı +æ¶ Ŀ +éŃĶ æľ¯ +éģĵè·¯ 交éĢļ +è§Ħ模 æľĢ大 +å®ŀæĸ½ æĦıè§ģ +ä¹ ŀ +ä¸Ģ ä¸ĸ +åŁ· è¡Į +è±Ĩ çĵ£ +åĪĹ ä¸º +æķħ 宫 +çĶŁ åij½åij¨æľŁ +ä¸īç§į èģĮä¸ļ +详ç»Ĩ ä»ĭç»į +å®Į å¤ĩ +岩 çŁ³ +éļı æīĭ +é£ ² +æķĪæŀľ åĽ¾ +ç§ĭ åĨ¬ +åĬŁ å¾· +è§Ħç«ł åĪ¶åº¦ +æĹ¥ æ¸IJ +æīĢ éľĢè¦ģ +æīĢéľĢè¦ģ çļĦ +å²Ľ ä¸Ĭ +åĩº åľŁ +åĽ¾ æĸĩ +ç§ijæĬĢ è¿ĽæŃ¥ +éĢļ èĥĢ +èĢģ 太太 +èĭĹ æľ¨ +éĵ¶ å·Ŀ +å¸IJ 篷 +éĿŀ è¦ģ +éħį çĶµ +å¤Ħ å¢ĥ +èĤ¡æĿĥ æĬķèµĦ +ä¸Ģ缴 åĪ° +åĿĩ çĶ± +æĬĹ æĹ¥ +æį® ä»ĭç»į +ä½ł åĸľæ¬¢ +åĪĽæĸ° åŀĭ +åıĺ è¿ģ +è§Ĩ å¯Ł +å®Įåħ¨ 没æľī +åħĥ æŦ +åı¯ ä¿¡ +åı¦ è¡Į +æĿij 级 +åħ¥ åľº +æIJŃ æ¡£ +ä¹Ł åĽłæѤ +æį¢ æĪIJ +ä¸į è´Ł +äºĨ 大éĩıçļĦ +éģĶ åĪ° +å¸Ĥ åİ¿ +å¹´ è¼ķ +å¿« æīĭ +å¸Į å°Ķ +èĩª èIJ¥ +éĽª èĬ± +æIJ ģ +çľ¼ ç§ij +æŃ£ 確 +çļĦ 姿æĢģ +åĿļå®ŀ çļĦ +æĮĩ 纹 +æªĶ æ¡Ī +ç½® äºİ +佩 æľį +豪 éŨ +åĵ Ĵ +æģ° 好 +檢 æŁ¥ +åĪĿ è¡· +大 åĶIJ +约 ä¼ļ +èĴ¸ åıij +çŃ¹ åĪĴ +å¹´ ç»Ī +è¡Į æ¥Ń +åħ± éĿĴ +åħ±éĿĴ åĽ¢ +ä¼ļ å¼ķèµ· +ä¸Ń ç§ij +ä¸Ńç§ij éĻ¢ +æĮ¯ åĬ¨ +åį´ åıijçİ° +ä¸įåĬ¨ 产 +èĮ ¹ +æĪ¿éĹ´ éĩĮ +è´§å¸ģ æĶ¿çŃĸ +æ²» çĻĤ +æħİ éĩį +å¡ŀ å°Ķ +åĽ½ ç±į +åĽł æŀľ +çŃī çī¹çĤ¹ +å±± è°· +ä¸ĭ è¼ī +è®ĵ æĪij +饮 éħĴ +è¿Ļ个 游æĪı +ç»Ŀ 大éĥ¨åĪĨ +åĴ¨è¯¢ æľįåĬ¡ +å¹² æ´» +è®® ä¼ļ +æ¦Ĥ è¿° +åĪĨ åĮº +æŃ» åIJİ +ç«Ļ çĿĢ +主è¦ģ é¢Ĩ导 +åIJĮ åŁİ +大 æłij +对 åѦçĶŁ +社ä¼ļ ä¿ĿéĻ© +å¢ŀ èµĦ +主人 åħ¬ +å®£ä¼ł æķĻèĤ² +æĸĩåĮĸ 交æµģ +客 æĪ¶ +çŁ¥åIJį åĵģçīĮ +æ»ŀ åIJİ +äºĴ è¡¥ +æĦŁ 人 +åī ¿ +åIJİ ä»£ +äºī 龸 +æķĻèĤ² åŁ¹è®Ń +éĿĻ èĦī +ä¹ı åĬĽ +说 åĩºæĿ¥ +çİĭèĢħ èį£èĢĢ +åĢ « +åįĩ èµ· +éķ ģ +åĩº 游 +éĢļè¡Į è¯ģ +å·¥ä½ľ å²Ĺä½į +åĮł å¿ĥ +æĭ¿ æĿ¥ +æ´Ĺè¡£ æľº +æĪijä¸į æĥ³ +é¢Ħ è§ģ +æ¼Ķ 示 +ä¸Ģ缴 没æľī +è·Ł 她 +对çħ§ æ£ĢæŁ¥ +ç° ¿ +ä¸ĵ å¿ĥ +è®® äºĭ +åīį 端 +åį¡ å°Ķ +è¨Ń å®ļ +设置 äºĨ +å©ļ 纱 +åľ¨ åĽ½å¤ĸ +åı³ 侧 +è³¼ çī© +å¥ĩ èij© +å¢ŀåĬł åĢ¼ +好 è¿IJ +åĽ½éĻħ æľºåľº +ä¸ĭ 称 +缮åīį 为æŃ¢ +ç¥ŀ ä»Ļ +å®ĥ åı¯ä»¥ +æ¾Ħ æ¸ħ +èĥ½ 使 +游 åĩ» +游åĩ» éĺŁ +åĩ ¹ +ä¸įè¦ģ åĨį +åĨ³ èĥľ +åĨ³ æĪĺ +æĭ ½ +缼 åħ¸ +å¾Ī好 åľ° +æľĢ ç¾İçļĦ +åĥ ļ +å·´ åŁº +å·´åŁº æĸ¯åĿ¦ +æľĢ éĢĤåIJĪ +é«ĺ èģĮ +ä¿Ŀ å§Ĩ +æİĪ æ¬Ĭ +说åĪ° è¿ĻéĩĮ +æݨ å¼Ģ +çİĩ è¾¾ +ä¸īåĪĨ ä¹ĭä¸Ģ +管çIJĨ ä¸Ńå¿ĥ +交 æ±ĩ +森æŀĹ åħ¬åĽŃ +å¾Ģ ä¸Ĭ +éªij è¡Į +æį® æѤ +纽 带 +ç» ŀ +ä¸ī æĸ¹ +æĦıä¹ī ä¸ĬçļĦ +æݨ è¿Ł +å¤ļæł· æĢ§ +æĥ³ èµ·äºĨ +æİĴåIJį 第 +å·¨ é¢Ŀ +æĿŁ ç¼ļ +å®ī å®ļ +äºĭ 實 +çļĦ æĦ¿æľĽ +è£ħå¤ĩ åĪ¶éĢł +人 å±ħ +人å±ħ çݯå¢ĥ +å¿ĺè®° äºĨ +该 游æĪı +楼 ä¸Ĭ +å¼Ģ ä¼ļ +æģ ³ +åıĭæĥħ éĵ¾æİ¥ +ç¡ Ĵ +ç»ĻäºĪ äºĨ +åģı 好 +åĵ ī +交éĢļ å®īåħ¨ +éĽ Į +æ²» çĹħ +è§īå¾Ĺ å¾Ī +衬 è¡« +å¿ĥ æĦ¿ +æ´ŀ å¯Ł +æ°ij æ£Ģå¯ŁéĻ¢ +æıIJ çĤ¼ +è¦ģ è¿Ľä¸ĢæŃ¥ +驾 车 +æĻ® æĥł +æķ ĸ +ç¦ı éŁ³ +éĢģ è¾¾ +è§ĦåĪĴ 设计 +æīĭ å¥Ĺ +å®ī ä¿Ŀ +è¿ĺä¸į å¦Ĥ +åīį è¿° +æłĩ è®° +ç´§ æİ¥çĿĢ +æ§ IJ +深深 åľ° +满满 çļĦ +æĺ¥ è¿IJ +æĹ¥ 产 +çĪ± æĬ¤ +åħ¨ æĹ¥ +åħ¨æĹ¥ åĪ¶ +转 åĬ¨ +ç¥Ń ç¥Ģ +ä¹° ä¸ľè¥¿ +对 æľªæĿ¥ +æ¶Ī失 äºĨ +åļ´ éĩį +ä¸ī æĿ¡ +éħ¸ 奶 +éĽĨåĽ¢ èĤ¡ä»½ +西 è·¯ +åıª å¾Ĺ +éĢģ åİ» +çĭł æĬĵ +åĪ©çĶ¨ çİĩ +ä¸ĭ åij¨ +å¥ĭ æĪĺ +æĺ¥èĬĤ æľŁéĹ´ +è´Ł 责任 +æĺĤ è´µ +å°¾ å·´ +ç¯ĩ æĸĩç«ł +åħ ® +è®Ĭ æĪIJ +å¹ ¹ +çĻ» éĮĦ +ä½ Ī +å·¥ åĮł +åĵªæĢķ æĺ¯ +åıį åĵį +ç§ ĥ +åĩº 轨 +æĹ¥ åĨĽ +åIJį èªī +æķı éĶIJ +æľįåĬ¡ æ°´å¹³ +çħ§ å°Ħ +ä¼Ĭ æĭī +ä¼Ĭæĭī åħĭ +åĨħ éĺģ +èĬĴ æŀľ +ä¸ĩ åĪĨ +éĢĢ 款 +缴æĴŃ éĹ´ +æĭ¿ åĪ°äºĨ +å°İ èĩ´ +空æ°Ķ ä¸Ń +客æĪ· æľįåĬ¡ +è¿IJ åĬ¿ +ç»ĵ çŁ³ +ä¸į å¿ħè¦ģçļĦ +èĥ¶ åĽĬ +çIJĨ ä¼ļ +æĬ½ åĩº +空æ°Ķ è´¨éĩı +æ¯ķ ç«Łæĺ¯ +åĨ· æ¼ł +ä¸Ģ å¦Ĥ +ä¸Ģå¦Ĥ æĹ¢ +ä¸Ģå¦ĤæĹ¢ å¾Ģ +æĤ£ çĹħ +åĬł æĮģ +èµŀ åĬ© +é« ® +åij½ ä¸Ń +æĦıä¹ī ä¸Ĭ +ä¸į èĪį +åģļ 梦 +æīĵ æī« +æĺŁ åħī +æĸŃ è£Ĥ +åħ¨ å¥Ĺ +è£ģ å®ļ +马 åħĭæĢĿ +骨 骼 +ä¸Ģ è·¯ä¸Ĭ +å®ļ æŶ +å·¥ç¨ĭ æĬĢæľ¯ +å½¼ å¾Ĺ +æ±² åıĸ +ä¸Ģ è§Ī +åIJµ æŀ¶ +ä¿Ĺ 称 +æłª æ´² +åºŁ æŧ +è¡Į æĺŁ +åıijçĶŁ åıĺåĮĸ +é¦ĸ ä»ĺ +åįģåĪĨ éĩįè¦ģ +æĬĬ è¿ĻäºĽ +ç¥ŀ å·ŀ +æıIJä¾Ľ åķĨ +æ¥ · +å± İ +çĬ¶ åħĥ +åŁİ å¢Ļ +çľĭ ä¸Ģçľĭ +çĶŁäº§ èĥ½åĬĽ +åŁºæľ¬ä¸Ĭ éĥ½ +æīĵ æī° +åĪĿ 次 +åĩº 示 +åħ¶ä¸Ń ä¸Ģ个 +çĶŁæĢģ ç³»ç»Ł +æīĭ æİĮ +æµİåįĹ å¸Ĥ +åľĭ åħ§ +æŃ£ åĢ¼ +å¹¾ ä¹İ +æݨèįIJ éĺħ读 +è¿Ń 代 +è°ĥ ä¾ĥ +饮 åĵģ +å¢Ļ ä½ĵ +åıĺ çİ° +äºĨ 好 +äºĨ好 åĩł +ä¸į çķĻ +çĪ ² +å°½ æĹ© +æŃ£åľ¨ è¿Ľè¡Į +åĩº éĻ¢ +æĿĢ 害 +æıIJ 款 +åıijå±ķ 空éĹ´ +åīį 身 +ä¸įæĸŃ å¢ŀ强 +æ·± å±Ĥ次 +容 纳 +éĤ£ 份 +å·¥ä½ľ æķĪçİĩ +æľ¬ åĽ½ +失 èIJ½ +æŃ£ åĽłä¸º +èĬĤ æ°´ +ä¸ĭ ä¸Ģ代 +çłĶåıij ä¸Ńå¿ĥ +ä¸į çIJĨ +å®Į 好 +ä¿ĿæĬ¤ åĮº +ç»ĵæŀĦ è°ĥæķ´ +å¥ł å®ļ +宣 称 +éĺ» æĮ¡ +æĴ¤ 离 +ä¸į æĸ¹ä¾¿ +åĴ ķ +ç¬ijäºĨ ç¬ij +çݯå¢ĥ 污æŁĵ +ä½ı æĪ· +ç»Ŀ ç¼ĺ +éĻ¤ å°ĺ +é«ĺ å°ļ +æĢİä¹Ī åı¯èĥ½ +éĿ¢ èī² +åķĨ æ¥Ń +çĸ ¹ +èµĦæºIJ ä¼ĺåĬ¿ +è¾ĸåĮº åĨħ +èĢĢ çľ¼ +æij§ æ¯ģ +ä¸ĸçķĮ ç»ıæµİ +å¼ķ æĿ¥ +ä¸Ģ åĪĻ +æĭĩ æĮĩ +æĬµ 御 +éĽ į +åĩĨå¤ĩ å·¥ä½ľ +çıł ä¸īè§Ĵ +ç¨Ģ åľŁ +èİ·å¾Ĺ æĦŁ +æĪIJåĬŁ çİĩ +ç½ij 约 +ç½ij约 车 +èĦ IJ +æķ¬ ä¸ļ +éĩij ä»· +ç²¾ é«ĵ +ä¹° 车 +åħ³ åı£ +åĨį å¤ļ +æŀģ åĵģ +åIJĦ 家 +举æĬ¥ çĶµè¯Ŀ +èļ Ĭ +æĸ¹ å½¢ +ç§ijæĬĢ æĪIJæŀľ +æľĢ好 æĺ¯ +éĹ® åĢĻ +红 éħĴ +åĽĽ ç§į +ç¿Ĵ æħ +ç¿Ĵæħ £ +åŀ ¦ +éĤ£ åıª +é¢Ĩ æĤŁ +çľ¼ éĥ¨ +æ³° å®ī +ä»» æľŁ +磨 æįŁ +æĽ¿ æį¢ +åħ¸ 礼 +符åIJĪ æĿ¡ä»¶ +è¿ĺæľī ä»Ģä¹Ī +åħ±äº« åįķ车 +åı¯ åĪĨ为 +åŃ£ åIJİ +åŃ£åIJİ èµĽ +举èİŀ å¸Ĥ +å¿ĥ æĦı +æīŃ æĽ² +ä½ľä¸º ä¸Ģç§į +è¿Ļ éĥ¨åĪĨ +åıĤä¸İ åĪ° +ç½ij çIJĥ +實 çı¾ +ç»Ħ è£ħ +åIJij å¤ĸ +å·¥ä½ľ æĸ¹æ¡Ī +åįģ æĿ¡ +課 ç¨ĭ +颤 æĬĸ +åĵ © +éĤ® å¯Ħ +äº ¢ +åħį è²» +ç§ ¤ +åºĶæĢ¥ 管çIJĨ +åĽĽ äºĶ +éºĴ éºŁ +å¾Ĵ æŃ¥ +è¨ĺ å¾Ĺ +çĴ IJ +æĺ¯åIJ¦ ä¼ļ +æĦıè§ģ åıįé¦Ī +éļ¾ æĢª +çª į +交 æİ¥ +两 åįĥ +æĩī çĶ¨ +æľŁ éĸĵ +æIJ¬ åĪ° +è®® é¢ĺ +碧 æ¡Ĥ +碧æ¡Ĥ åĽŃ +åģļ çĶŁæĦı +éĻĽ ä¸ĭ +è· ĭ +èĢģ人 家 +带 åĽŀ +æŀ¸ æĿŀ +è¡Į éķ¿ +åĨħ容 ç®Ģä»ĭ +æ¢ ¢ +æĮĩ æݧ +éĩį çĹĩ +ç½ijåıĭ 们 +çı¾ 代 +ç±» 产åĵģ +å¥Ķ æ³¢ +æ¸ º +ç²ī ç¢İ +è¿Ļ åıªæĺ¯ +æ£Ģå¯Ł æľºåħ³ +é½ Ĭ +æĪ¿ 秣 +å¾· æĭī +å²ģ 以ä¸Ĭ +纯 åĩĢ +åĪĨå¸ĥ åľ¨ +èĥ½ å¾ĹåĪ° +ä¸į å°½ +ç«ŀ ä»· +çļĦ 带é¢Ĩ +çļĦ带é¢Ĩ ä¸ĭ +ä¸Ńèį¯ æĿIJ +æĿij éķĩ +ä¸įåı¯ éģ¿åħį +éľ² 天 +å°ı å§ijå¨ĺ +çī© 件 +èijĹä½ľ æĿĥ +æĭĺ çķĻ +éĥ½ è§īå¾Ĺ +æĽ² æĬĺ +æ·»åĬł åīĤ +åı¬ åĽŀ +æīİå®ŀ æİ¨è¿Ľ +æĬĦ è¢Ń +åĮĸ 身 +缴 èIJ¥ +ä¹Ł å¸ĮæľĽ +èį£èªī 称åı· +åįĸ ç»Ļ +æľī ä¸įåIJĮçļĦ +å¥ĩ çī¹ +éĥ½ 认为 +å¦ ŀ +æĪIJéķ¿ 为 +辩 æĬ¤ +主 æķĻç»ĥ +æ³ķå¸Ī èģĮä¸ļ +æ¤į åħ¥ +ç´¢ å°¼ +åIJ¬ è¿ĩ +ä¹łæĥ¯ äºĨ +夺 åıĸ +éŁ ĵ +æľ¬è´¨ ä¸Ĭ +æİ¥ åĬĽ +äºij 端 +è¦ģ åģļ好 +è·¯ çģ¯ +åįıåIJĮ åıijå±ķ +æľī å¾ħ +æ°´ åŁŁ +æIJľçĭIJ é¦ĸ页 +è´¨éĩı å®īåħ¨ +åįģäºĮ äºĶ +åĵ® åĸĺ +èĵ¬åĭĥ åıijå±ķ +åIJį 声 +身 亡 +çİĭ åºľ +åİŁåĪĻ ä¸Ĭ +çĥĺ å¹² +éģĹ æ¼ı +éĿ¢ 缮 +åĽ½ ä¼ļ +ä¸Ģ缴 éĥ½æĺ¯ +æľīä¸Ģ ä½į +éħį æľī +éĻª çĿĢ +ä¼ģ åĽ¾ +æĮī ä¸ĭ +èĵĿ åĽ¾ +æ© ĺ +大å¤ļ æĺ¯ +辩 论 +æĹĭ å¾ĭ +æĬ¥ éĢģ +æĿ¡ è§Ħå®ļ +åĬ¨ éĿĻ +åĮĪ 奴 +æĭľ 访 +ä¸Ģ åĪĢ +ä»ĸ çŁ¥éģĵ +主 æĿĥ +ä»ĸ æĽ¾ +æĴŃ ç§į +å£ģ åŀĴ +çī¢è®° 使åij½ +åľ¨è¿Ļ æĸ¹éĿ¢ +æīĭ èħķ +æĶ¯ æŀ¶ +ä¾Ĩ èĩª +éĩį å¡ij +å¤ļ å±Ĥ次 +ä»ĭ è´¨ +éĿ¢ åŃĶ +æ½® 湿 +åİ¿ åŁŁ +游æĪı å½ĵä¸Ń +å£ ŀ +åĪĹ åĩº +èµĽ åĮº +å¤ļ åįĬ +éĩįçĤ¹ å·¥ä½ľ +æĪij们 å¿ħé¡» +æŁı æŀĹ +é²ģ èĥ½ +æĸ½ å±ķ +åIJĦ åĮº +åħį ç¨İ +èµĽ åIJİ +æľĢ éĩįè¦ģ +ä¸Ģ个 好çļĦ +è¿Ŀæ³ķ è¿Ŀè§Ħ +äºĨ解 æĽ´å¤ļ +æķ¬ 请 +ç¬ijçĿĢ 说 +ä¸įæĸŃ åıijå±ķ +æijĦå½± å¸Ī +以 éĺ² +çĤ¸ å¼¹ +声 åĵį +ç¤ ģ +æĩ ¿ +èĪĨ æĥħ +èĩªçĶ± è´¸æĺĵ +æķı æį· +ä¸ī大 éĺ¶æ®µ +èĭ Ķ +æĹº åŃ£ +ä¸į 满æĦı +微信 åı· +ä¿® 为 +çł´ è£Ĥ +éĢĥ 离 +æ¯ı èĤ¡ +è¾¾ ä¸įåĪ° +æ¯ıå¹´ éĥ½ +çģ¯ 笼 +æѤ åŁºç¡Ģä¸Ĭ +åĥı 个 +åĪĨ 娩 +æĻ ¾ +ä¸į èĩ³äºİ +红 线 +误 解 +举 è·¯ +æ·® å®ī +产 åѦ +产åѦ çłĶ +èī¾ æ»ĭ +èī¾æ»ĭ çĹħ +åīįæıIJ æĺ¯ +æ¯ı ä¸Ģ天 +ä¸ĥ 大 +æłij åı¶ +èµ° å¾Ĺ +è¿Ļ 两ç§į +æİı åĩº +æİ IJ +é¢Ĩ导 èĢħ +ä¸Ģ æľµ +个å¤ļ æľĪ +ä¸Ń åħ³ +ä¸Ńåħ³ æĿij +课åłĤ æķĻåѦ +大 åĴĸ +éģĭ çĶ¨ +è¯ļ æĦı +ç»Ħ åĽ¾ +è¯ķ çĿĢ +ä¹Ķ æ²» +è¿ĺ ä¸įæĺ¯ +æľī æĽ´å¥½çļĦ +åIJİ å¤ĩ +æĸ°çĶŁ åĦ¿ +æ°Ķ è¡Ģ +æ²¥ éĿĴ +å±ı éļľ +æ¥Ń åĭĻ +æĪij 以为 +éķ¿ 缸 +èĢģ çĪ¸ +éķĩ æ±Ł +æľºæ¢° 设å¤ĩ +ä½Ĩæĺ¯ å¦Ĥæŀľ +åĿļå®ļ ä¸į +åĿļå®ļä¸į 移 +åĨ² éĶĭ +ç®Ģ缴 æĺ¯ +åĤ¨ èĵĦ +纯 çĶµåĬ¨ +漫 æŃ¥ +举 èµ· +æģ¶ æĢ§ +è¨ĺ éĮĦ +èģĮèĥ½ éĥ¨éŨ +åħ¨ éķ¿ +鼻 è¦ĸ +ä¹³ èħº +ä½ķ å¤Ħ +æ¶Ī æŀģ +æŃ£ å¤Ħäºİ +å®ī å®ģ +æĪIJ éķ· +åıĻ è¿° +æºĥ çĸ¡ +ä½Ĩ çİ°åľ¨ +女 æĺŁ +å©´ å¹¼åĦ¿ +æĬķ èŀįèµĦ +éĹ® éĹ® +æıŃ å¼Ģ +è¯ ı +åIJį å½ķ +èĺij èıĩ +åIJĬ 顶 +æ¹ĸ åĮº +åįĸ åľº +建 ç¯ +å»ºç¯ ī +èİ ½ +åIJ¬ åIJ¬ +ç«ŀäºī ä¼ĺåĬ¿ +åĩº ä»» +æľī 两ç§į +橱 æŁľ +è¤ ª +è¯ķ åį· +ç»ıæµİ æĬĢæľ¯ +æ·± å±Ĥ +éĩįè¦ģ åĨħ容 +é£İ æݧ +çĬ¶æĢģ ä¸ĭ +éĥ¨ éĸĢ +广 æ±½ +è§Ĥ æij© +éģĹ çķĻ +转 è´¦ +æĮģ ä»ĵ +æĢ» 计 +åľĺ éļĬ +æĪ¿ 举 +éĺĢ éŨ +åħ¬ åħ³ +åħ³ åĪĩ +èĤ ĺ +æķ¸ æĵļ +ä¸ī åįģå¹´ +è§ģè¯ģ äºĨ +å± Ĩ +çģ° å°ĺ +æ¦ľ é¦ĸ +è¦ĨçĽĸ çİĩ +ä»Ļ 女 +çĶŁäº§ æĢ» +çĶŁäº§æĢ» åĢ¼ +æĪ¿ è´· +æ±Ł åĮº +åħħçĶµ æ¡© +çĻ¾ åIJĪ +確 èªį +转 移åĪ° +éĥ½ æĹłæ³ķ +纪念 é¦Ĩ +çŃ¾ç½² äºĨ +并ä¸į å¤ļ +æĮ ł +ä¸į太 好 +ä¸ĸ 代 +误 导 +é«ĺå³° 论åĿĽ +åħ¼ 容 +龸 æ°Ķ +æĿ¥ 访 +æīĢ 带æĿ¥çļĦ +æĺ¯ä¸Ģ éĥ¨ +æĻļ é¥Ń +åİĨ 代 +åIJ¦ åīĩ +ä¹ħ ä¹ħ +æľīæķĪ æľŁ +诱 åıij +æĢ» èµĦ产 +æľ¬èº« å°±æĺ¯ +çĶŁäº§ åİĤ家 +æŶ 髦 +èĢIJ çĶ¨ +ä»İå°ı å°± +æĿ¡ 约 +èĭ± åĭĩ +ä¿Ĺ è¯Ŀ说 +寺 åºĻ +å¿ĥçIJĨ åģ¥åº· +ä»Ģä¹Ī äºĭæĥħ +æ±ī åŃĹ +çķĻ ä½ı +åįĹ è·¯ +ä¸ī 项 +丢 äºĨ +æĥ³ åĪ°äºĨ +çŃ¹ éĽĨ +éĻĦåĬł åĢ¼ +西 è£ħ +ä¹ĭ ä½ľ +åģļçļĦ äºĭ +çķ¶ æĤ¨ +çķ¶æĤ¨ åľ¨ +é¦ĸ 款 +ä¸įåľ¨ ä¹İ +å·¥ç¨ĭ æĸ½å·¥ +éļIJ éļIJ +åıĺ 身 +沿 éĢĶ +æĤł æĤł +ä¿Ŀ æļĸ +çĶŁæ´» åŀĥåľ¾ +渤 æµ· +æѦ ä¾ł +女 主è§Ĵ +举 ä¾ĭ +æ ·¨ +çĻ½ é¢Ĩ +è£Ļ åŃIJ +è¿Ķ è¿ĺ +è¿Ī åĩº +é¾Ļ éŨ +ç»ıæµİ ä½ĵ +æĶ¶ å®ĺ +çķĮ éĻIJ +è·³ åĩº +åįĩ åĢ¼ +绵 éĺ³ +çĸ¤ çĹķ +çľĭ æ¸ħ +æĭĴ çµķ +è¥Ħ éĺ³ +课 å¤ĸ +åŃIJ åŃĻ +æŃĮ è¯į +æĪIJ åIJį +溶 液 +åĦĴ 家 +åķĨä¸ļ åĮĸ +辨 åĪ« +å¤ļ è¾¾ +ç½ij åºĹ +ä¹Ŀ 大 +ä¹Ŀ大 ç²¾ç¥ŀ +æѤ 举 +è¿ŀ è½½ +ä¸Ģ åĢĭ人 +èī² æ³½ +涵çĽĸ äºĨ +è¦ı åĬĥ +åĽ½ æĥħ +åį«çĶŁ åģ¥åº· +积æŀģ åĵįåºĶ +æĭ Ļ +åĪ¶ åĬ¨ +æĥ³è±¡ åĬĽ +çļĦ ä¹IJ趣 +å¼łå®¶ çķĮ +å´ İ +éĩį åŀĭ +å¤ĸ å¢Ļ +æĶ¾ åѦ +è®¤çľŁ åŃ¦ä¹ł +è´¬ åĢ¼ +æ³ķ æ¡Ī +æĬ¤èĤ¤ åĵģ +éĻ·åħ¥ äºĨ +请 æĤ¨ +åŀ ¢ +æķĻèĤ² èµĦæºIJ +交æĺĵ å¹³åı° +æŶ è£ħ +ä¼łæŁĵ çĹħ +æ¹ĸ æ³Ĭ +èµĦ 管 +åݨ å¸Ī +éĹľ éį +éĹľéį µ +åĵĪåĵĪ åĵĪ +çĽĹ çªĥ +çĶľ ç¾İ +åºĦ åĽŃ +缮åīį å·²ç»ı +è¾¹ ä¸Ĭ +çģ« èĬ± +æĬ¥ è®°èĢħ +æģĭ æĥħ +ç´§ åĩij +æ°´ æµģ +è¿Ļæĺ¯ æĪij们 +æ³¥ åľŁ +æĽ¾ ä»» +æĸ¹ è¨Ģ +åij¨ åħŃ +åı· 楼 +ä¼ij åģĩ +误 ä¼ļ +åĽ½ åĢº +åīį å¤ķ +两 å¼ł +éĹ « +éŃĶ 鬼 +æĬĬ æĮģ +èĬĤèĥ½ çݯä¿Ŀ +æ¸ħæ´ģ èĥ½æºIJ +èĤ¥ æĸĻ +é«ĺ é¢ij +å°± æľīäºĨ +交 ä¼ļ +没 éĴ± +éĽħ æĢĿ +è¦ģ åıĬæŶ +åŁ¹åħ» åѦçĶŁ +欣 åĸľ +çĥŃæ°´ åĻ¨ +é¾Ļ æ¹ĸ +äºĮ 楼 +æĸ°æµª è´¢ç»ı +æĸ° åĬ¨èĥ½ +èµ£ å·ŀ +æĭ³ 头 +æµģ åIJij +ä¹Łæĺ¯ å¾Ī +åıij åĶ® +ä¸Ń åIJ«æľī +åIJĵ å¾Ĺ +å·¨ æĺŁ +æĹł æīĢè°ĵ +æ¯Ľ åŃĶ +åħ¬åħ± 交éĢļ +çĤİ çĥŃ +èµ· èįī +åĬłçĽŁ åķĨ +说 ä¸įåĩº +大åѦ æ¯ķä¸ļ +å·¥ä¸ļ åĽŃ +éłĺ åŁŁ +åºĨ åħ¸ +æµģ 产 +èģ² éŁ³ +ä¼¼ä¹İ æĺ¯ +è´§ æºIJ +æ·± åĪĩ +æ²»çĸĹ æĸ¹æ³ķ +èµĦæºIJ éħįç½® +網 åıĭ +çĶ £ +äº ¥ +躲 åľ¨ +社 ç§ij +è»Ł é«Ķ +女 è£ħ +æŃ¡ è¿İ +综åIJĪ å®ŀåĬĽ +æł¼ å°ĩ +åħļåı² åŃ¦ä¹ł +æľĢ åŁºæľ¬ +æľĢåŁºæľ¬ çļĦ +çľĭ æľĽ +åıĹ è´¿ +ä¸įä»ħ èĥ½ +ä½ķ å¿ħ +ä¸Ģ个 å°ıæŶ +ç¾ Į +æĭĽ æĶ¶ +çĤĴ èĤ¡ +æĿij å¹²éĥ¨ +缸 çĪ± +æ½ľ èĥ½ +ä¹ į +æŶ è¾° +欣 æħ° +éĵ¶ è¡Įä¸ļ +çĭŃ çªĦ +éĩįçĤ¹ é¢ĨåŁŁ +çİ°å®ŀ çĶŁæ´» +éĮ¯ 誤 +æĸ° è§Ħ +滥 çĶ¨ +æŶ ä¸į +æŶä¸į æŶ +帳 èĻŁ +ç¨Ģ 缺 +åIJij 举 +ä¿Ŀåģ¥ åĵģ +çıŃ éķ¿ +äºĴ åĭķ +笼 罩 +æ½ Ľ +æļĸ å¿ĥ +è½° çĤ¸ +åºĨ 幸 +è²Į ä¼¼ +æĵ º +èĢIJ 磨 +ä¸ĵä¸ļ 人士 +ä¸ĢèĪ¬ éĥ½æĺ¯ +æ¼³ å·ŀ +åħ¨ èĩªåĬ¨ +å½ķ çĶ¨ +大 è·Į +æľīæķĪ æĢ§ +èĩª åĭķ +ä¸ī个 æĸ¹éĿ¢ +港 åĮº +ä¿¡ 貸 +éĢļ è¯Ŀ +é«ĺ 涨 +æ³Ħ æ¼ı +éħį ä¸Ĭ +åħļ å·¥å§Ķ +被 认为 +被认为 æĺ¯ +ä¸įä¼ļ åĨį +è°ĥ åīĤ +åıĤ èĤ¡ +èĦ± åıij +å¿ł å®ŀ +åĨħ åĪĨæ³Į +ç¹ģ å¿Ļ +åıĮ åĪĽ +é©» æĿij +åĪĴ ç®Ĺ +éģİ ä¾Ĩ +åľ£ ç»ı +èıľ 鸣 +æĭ¼ å¤ļå¤ļ +ä¸ŃåĽ½ 汽车 +çĥŁ èįī +缴 æµģ +äºĨä¸Ģ åı£æ°Ķ +ä½İ æĪIJæľ¬ +æī¾ åĽŀ +èĩª åįij +總 æĺ¯ +æĸĩåĮĸ åĪĽæĦı +天 æ²³ +樱 æ¡ĥ +éªij åħµ +éĩĮéĿ¢ æľī +çİ ® +èĥ½ æī¾åĪ° +éĢĥ è·ij +åĪĩ å°Ķ +åĪĩå°Ķ 西 +以ä¸ĭ æĺ¯ +å²³ éĺ³ +çļĦ æ¦Ĥçİĩ +æĬµ åĪ¶ +å¸Ī äºĭåĬ¡ +å¸ĪäºĭåĬ¡ æīĢ +åĩĨ æŶ +屬 æĸ¼ +订 è´Ń +åįłæį® äºĨ +ä¸Ń éĢĶ +å° ĭ +é»ij 马 +åİ¿ åħ¬å®īå±Ģ +ä¸ĥ æľĪ +èī² ç´ł +å¿ĥèĦı çĹħ +æŶ éĻIJ +æ¯į åħ¬åı¸ +å¹ķ åIJİ +ä¸Ĭ æ¦ľ +åĢ¾åIJij äºİ +纸 ä¸Ĭ +æ¡ ĵ +éĽĨä½ĵ ç»ıæµİ +æĥħ å¢ĥ +è¦ģ åģļåĪ° +ç©į 極 +åıª æĢķ +æ¹ĺ 西 +çļ± 纹 +åħ¨ åľĭ +çĦ¡ è«ĸ +好 æĦŁ +åįķ ä»· +è¿Ľç¨ĭ ä¸Ń +æĺĨ ä»ij +åĪĽ 客 +åħħ æĸ¥ +åħĪ æĬĬ +该 æĢİä¹ĪåĬŀ +åĵģ å¾· +åħ¨éĿ¢ åıijå±ķ +è¨Ī åĬĥ +æĢ» å·¥ä¼ļ +ä½Ľå±± å¸Ĥ +æĬĹ è¡¡ +å¼Ģ åľº +éĴ± å¸ģ +åıĭ 们 +å«ī å¦Ĵ +ç´¢ èµĶ +è®Ĭ åĮĸ +æĮ¤ åİĭ +æĮij è¡ħ +çŃī ä¸Ģæī¹ +æĿ¨ 欢 +ä¸ĵ家 åѦèĢħ +èĥ½ è¾¾åĪ° +èµ° è¿ij +è´«åĽ° åľ°åĮº +éĻIJ æľŁ +ä¸į 平衡 +åĽ½åĨħ å¸Ĥåľº +èµĽ åľº +éħį èµĦ +è¦ģ èĢĥèĻij +ä¸ĩ åı° +æľĪ æľ« +éĶ ¥ +åŃ « +æݥ触 åĪ° +åĩº 产 +æķĻ åѸ +ä½ľ å¼Ĭ +çļĦ æľĢåIJİä¸Ģ +ä¿ĥ æĪIJ +åIJ¸ åıĸ +æ½ľ èīĩ +被 éªĹ +è¾ĵ äºĨ +çĭIJ çĭ¸ +åįĩ éĻį +è¿ĻäºĽ ä¸ľè¥¿ +æĬķèµĦ åŁºéĩij +çĶŁçī© åѦ +ç½ij绾 èIJ¥éĶĢ +åIJij è®°èĢħ +èįī åľ° +æĢ ¯ +æľįåĬ¡ èĥ½åĬĽ +éĥģ éĹ· +åįķ åĵģ +å¾Ĺ 罪 +æĺĵ äºİ +个å¤ļ å°ıæŶ +éĩį ä»» +ä¸Ĭ å®ĺ +æľ¬ éĩij +çı¾ åł´ +溢 ä»· +æĺŁ è¾° +æ´»åĬ¨ çİ°åľº +丹 麦 +å¸Ŀ çİĭ +æŁ¥ æĺİ +åŃĺåľ¨ äºİ +é¦Ļ æ°´ +æĬ½ æ£Ģ +å®ŀéĻħä¸Ĭ æĺ¯ +æĸ° å¾ģç¨ĭ +è´¢åĬ¡ 管çIJĨ +æİ Ľ +åĨľ åİĨ +éĥ½ èĥ½å¤Ł +éĤ¯ éĥ¸ +羣 實 +ç» Ĭ +åĨµ ä¸Ķ +ç½® 身 +ç¥Ī 祷 +çĿģ å¼Ģ +æĮĩ çĤ¹ +å¼Ģ æľº +西 å®ģ +åĮĹ çº¦ +积 æ°´ +åĩº åĬ¨ +åıijå±ķ 模å¼ı +转 æĬĺ +èĢĥ çĤ¹ +æľī ç½ijåıĭ +è´«åĽ° æĿij +æĪij们 çŁ¥éģĵ +åĪĨ éĶĢ +å±± èĦī +æ¯Ķ æĭŁ +ä¼° ç®Ĺ +æĶ¹ 建 +壮 è§Ĥ +ç§ī æĮģ +æı ª +ç¦ Ģ +åĮĸåѦ åĵģ +ä¸ŃåĽ½ åĪ¶éĢł +ä¸Ģ æŀ¶ +æīį è¡Į +æĭĽ å¾ħ +åıĺ æį¢ +åīį 线 +幸 好 +è¿Ļæł· çļĦè¯Ŀ +å¿ĥ è¡Ģ管 +æĢ§ çĸ¾çĹħ +åħ¨ èĥ½ +åĪij 侦 +ä¿¡æģ¯ åıijå¸ĥ +æĺ¾ çĦ¶æĺ¯ +éĿĴ éĵľ +åIJĥ ä»Ģä¹Ī +çĶµ ä»· +æ³ķå¾ĭ è§Ħå®ļ +çħ ² +çĵ· åĻ¨ +èĤī ç±» +æıĴ åħ¥ +åĹ ľ +è¿Ł è¿Ł +ä¸ĢçĤ¹ éĥ½ä¸į +è¿ĺ åĮħæĭ¬ +èĪį ä¸įå¾Ĺ +æłĩå¿Ĺ æĢ§ +æľĪ 以æĿ¥ +ç³ĸ æŀľ +éĥ½ åºĶ该 +çݯå¢ĥ åį«çĶŁ +èĪª è¡Į +éĥij éĩį +ç½ij æĬķ +åįģ ä½³ +ç§ģ ä¸ĭ +æļ´ è·Į +åĬłå¿« åıijå±ķ +产åĵģ çłĶåıij +åĪĽéĢł åĩº +æĢ» è§īå¾Ĺ +åºķ çĽĺ +èķ Ĭ +åĩºå¸Ń ä¼ļè®® +主 æĿ¿ +æĹ¥æĻļ éĹ´ +å®ĺæĸ¹ å¾®åįļ +å¼ķçĶ¨ æĹ¥æľŁ +åī¯ æķĻæİĪ +çĶµåŃIJ 产åĵģ +è¡° éĢĢ +çķĻ åŃĺ +çģ« åĬĽ +çĴ § +çļ Ĥ +åħ¼ åħ· +éĩį è¿Ķ +é¢Ĩ çķ¥ +åĪĩ éĻ¤ +åĨįçĶŁ èĥ½æºIJ +å®ŀåľ¨ 太 +çIJĨ论 ä¸Ĭ +ä¸ī å±Ĥ +ä¸ĸçķĮ åIJĦåĽ½ +å®ľ æĺĮ +èĢ³ è¾¹ +宽 æķŀ +æ±ī æĹı +çĻ½ çĻ½ +è¿ĻéĩĮ éĿ¢ +çĶŁæ´» ä¹łæĥ¯ +èµŀ èµı +çĶ· 士 +ä¸Ń ä¿Ħ +车 祸 +åīĤ éĩı +éĻ¤ åİ» +å·¦ è¾¹ +çŃij çī¢ +çīĽ å¸Ĥ +家 åĬ¡ +åķ ĥ +ç½® æį¢ +ç´« å¤ĸ +ç´«å¤ĸ 线 +å¾Ģ åīį +åĬĽ åѦ +ç´§ è·Ł +缮çļĦ åľ¨äºİ +ç» ® +ç¥ Ĥ +宣 è¨Ģ +äºĮ æ°§åĮĸ +äºĮæ°§åĮĸ 碳 +æĹł ç¼ĺ +ç²¾ éĢļ +è¨ º +å¼ķåıij äºĨ +æľĢ åħĪ +æ´¾ é©» +ä¸į å¿į +æĪij çĪ¸ +å¹´ ä¸ĭåįĬå¹´ +æ·ĭ å·´ +没 éĹ®é¢ĺ +åºĹ åĨħ +è·Ł æĪij说 +çĶŁäº§ çĶŁæ´» +è§Ĥ æľĽ +æ¸ į +被 æī§è¡Į +被æī§è¡Į 人 +èĪ ľ +æİ º +ä¸Ģ ç§Ĵ +èįī åĿª +åij¼ åĴĮ +åij¼åĴĮ 浩 +åij¼åĴĮ浩 çī¹ +人æ°ij éĵ¶è¡Į +çĦķ åıij +è¯ģåĪ¸ 交æĺĵ +çķ Ķ +æľº èĥ½ +å¦ ¾ +æĻļ å¹´ +å·¥åķĨ èģĶ +åİŁ åŀĭ +è§Ĵ度 çľĭ +æĬ¥ 社 +è¯į æĿ¡ +躲 éģ¿ +éĩį åIJ¯ +å¤ķ éĺ³ +èĤ¡æĿĥ 转让 +åľ¨ ä¸Ģ +åľ¨ä¸Ģ æĹģ +社ä¼ļ åĮĸ +åıijå±ķ åİĨç¨ĭ +æĭĸ æ¬ł +使 èĢħ +ä¸İ åIJ¦ +æĸ° å±ĢéĿ¢ +ä»Ĭ天 æĪij们 +é½IJ èģļ +对 æĪij说 +éĢĴ 交 +æľª æĽ¾ +èİ Ĭ +éĸ ī +亲 æīĭ +è§Ĵ éĢIJ +æľī é»ŀ +ç¨İ çİĩ +ä½İ 声 +é»ĺ å¥ij +æĻ® æ³ķ +大 ä¸ĵ +第äºĮ 大 +ä½ı åĿĢ +æĶ¾ è¿Ľ +äºĮ æĪĺ +亲 身 +åĽº åĮĸ +ä¸ĭ 乡 +åħ³éĶ® æĬĢæľ¯ +åĽŀ æĥ³ +æĬ¥ åĪĬ +æ¶Ĥ æĬ¹ +èĹı çĿĢ +ç¥Ŀ æĦ¿ +åįĩ 温 +çĶļèĩ³ è¿ŀ +åħ¬åħĥ åīį +ç¾İ æĸ¹ +è¯ļ å®ŀ +æĹł åģ¿ +åīµ æ¥Ń +å°ıå¿ĥ 翼 +å°ıå¿ĥ翼 翼 +两 æīĭ +温馨 æıIJ示 +仿 羣 +æĥ ¶ +èĥ¡ åŃIJ +å·¥ä½ľ ç«Ļ +硬 çĽĺ +ç« ¿ +åĤ³ éĢģ +åħ¨ æł¡ +é²ľ æ´» +çĴĢ çĴ¨ +ç»ĵ å°¾ +æį¢ æĿ¥ +æĪ Ģ +ä½İ ä½į +ä¸ĩåħĥ 以ä¸Ĭ +åĬł åĪĨ +æݨä»ĭ ä¼ļ +çIJĨ èµĶ +å¾· å°Ķ +æĬĹ è®® +æ´ ¼ +åĸ § +åŁİ éĻħ +å¾Ī æ£Ĵ +人 æŃ»äº¡ +ä¼ļå±ķ ä¸Ńå¿ĥ +äºĴèģĶ äºĴéĢļ +èĸĦ èĨľ +éĩį é»ŀ +ç¦ģ æ¯Ĵ +åĨ· ç¬ij +大家 åı¯ä»¥ +é¦ĸ 缸 +è¿ij è·Ŀ离 +æµ® çİ° +ç§ĺ è¯Ģ +èµ· é£ŀ +æIJ ¶ +羣 åģĩ +æģ ķ +å°ı åºĹ +æ°ij çľ¾ +åıijå¸ĥ åħ¬åijĬ +侧 éĩį +å¾ĺ å¾Ĭ +æĢ Ķ +æª IJ +æķ° 缮 +åī¯ ç§ĺ书éķ¿ +两 åı¥ +éļIJ çŀĴ +åıĮ åıĮ +æīĭ æĦŁ +èij¡ 京 +éģĹ å¿ĺ +é¬ ¥ +è¿Ļ个 åľ°æĸ¹ +说 çļĦè¯Ŀ +å·¡ åĽŀ +è¿Ŀ ç«ł +æī¾ å·¥ä½ľ +æĶ¯ çIJĥéĺŁ +裡 éĿ¢ +æĺ¾ç¤º åĩº +èĩ³ å°Ĭ +两 级 +åīį 段æŶéĹ´ +çĺ¦ èº« +èĤ¢ ä½ĵ +æ¯į 親 +æīĭç»Ń è´¹ +汽车 è¡Įä¸ļ +æİ© çĽĸ +æݧèĤ¡ éĽĨåĽ¢ +åı£ å¾Ħ +æĶ¿çŃĸ æݪæĸ½ +æµ· 绵 +åħ¨ éķĩ +äºĭ åħ³ +å¸Ń æī§è¡Į +å¸Ńæī§è¡Į å®ĺ +éĤ£ 次 +åı¯èĥ½ åĩºçİ° +ä¸Ńå¿ĥ åŁİå¸Ĥ +ç¿» 身 +ä¹Ł ç®Ĺ +ä¾µ çķ¥ +åĸĩ åıŃ +æ¯ı次 éĥ½ +è§ ħ +éĻ¢ éĻ¢éķ¿ +å§ĭ äºİ +èѦ åĬ¡ +èį¯ æĿIJ +å±ł æĿĢ +æľ¬èº« å°± +éļıæŶ éļı +éļıæŶéļı åľ° +åĶ® åįĸ +æĹłäºº 驾驶 +é¢ ħ +åĵģ 質 +åĺ² ç¬ij +è·ij åİ» +åħĭ éĩĮæĸ¯ +çķ¸ å½¢ +ä¿® 饰 +磩 éĺµ +éŁ³ä¹IJ ä¼ļ +æŁ³ å·ŀ +é½ ¡ +ä¼ļ è°Ī +æŃ£ çīĪ +ä¹Ł åIJĮæł· +æļ§ æĺ§ +è¡ĮæĶ¿ éĥ¨éŨ +ä¹ĸ ä¹ĸ +èĤ¤ èī² +æŶ ä»» +羣 åĪĩ +æľĪ ä¸ĭ +æľĪä¸ĭ æŬ +举æĸ¹ è´¢å¯Į +è£ħä¿® åħ¬åı¸ +éĢĢ è¿ĺ +åĭĺ å¯Ł +åĵ¥ 伦 +åĵ¥ä¼¦ æ¯Ķäºļ +çĭ¬ ä¸Ģ +çĭ¬ä¸Ģ æĹł +çĭ¬ä¸ĢæĹł äºĮ +è°ĥ åij³ +åİĭ è¿« +åħ¨çIJĥ æľĢ大 +åī¯ æł¡éķ¿ +æĽ´ ä½İ +åĪĨéĴŁ åIJİ +åĽŀ ä¾Ĩ +åĪ¶ åīĤ +åijĬè¯ī 大家 +çĤ¹ éĴŁ +åįģä¸ī å±Ĭ +åij¨ åĽĽ +è¿Ļæł· ä¸Ģ +è¿Ļæł·ä¸Ģ æĿ¥ +èĭ Ł +æľĽ åİ» +æĪIJ è¯Ń +å½ĵ åį³ +ç¬ij 声 +ä¹ĭ åĬ¿ +åĪijäºĭ æ¡Ī件 +æĮĤ çĿĢ +ä½ķ ç§į +å°ı 游æĪı +åĽ½å®¶ æĪĺçķ¥ +åĨ· åĨ· +å®ľ 宾 +æIJº ç¨ĭ +è¶ĭ äºİ +åıį çľģ +常 说 +ä¸ĩ æĪ· +åĥµ å°¸ +åįĥä¸ĩ åĪ« +åıijçİ° éĹ®é¢ĺ +åı¯ çŁ¥ +éŨæĪ· ç½ijç«Ļ +åģ¥åº· 产ä¸ļ +åı³ è¾¹ +æµ· è¿IJ +è¿ij ä¹İ +åĮ» æ²» +æĢ» ç®Ĺ +ä¸Ģ åĪĨéĴŁ +æĭ § +ä¹Ł æľīä¸ĢäºĽ +ä¾ĽçĶµ åħ¬åı¸ +å»ī ä»· +帮 ä»ĸ +æŃ¤æ¬¡ æ´»åĬ¨ +åıªèĥ½ 说 +èĬ ĭ +çīĩ 段 +åŃĺåľ¨ éĹ®é¢ĺ +ä½łä¼ļ åıijçİ° +è½® å»ĵ +ç½ij éĢļ +滨 æ±Ł +æİĪ ä¿¡ +é»İ æĺİ +ä¸į å±ŀäºİ +约 åįł +éķ¿æ²Ļ å¸Ĥ +èĥļ èĥİ +åħĥ 件 +éĻĨ åĨĽ +è³¼ è²· +æĮĩ æľĽ +å®ŀä¹ł çĶŁ +çī¹çĤ¹ æĺ¯ +çıł æ±Ł +çľĭ ä¸įåĩº +ä¸įè§ģ äºĨ +ç¼ ī +éĺµ èIJ¥ +åĶIJ æľĿ +没 å¿ħè¦ģ +åĽ½åľŁ èµĦæºIJ +ç»ıæµİåѦ 家 +åIJĪèĤ¥ å¸Ĥ +çIJ¢ 磨 +ç¡® åĪĩ +åŁİå¸Ĥ åıijå±ķ +çŃ· åŃIJ +人æ°ij æľįåĬ¡ +满 åĪĨ +è¿· ä¿¡ +ä½ľèĢħ æľ¬äºº +æĸĩç«ł æĿ¥æºIJ +ç«Ļ ç«ĭ +æŀĦ æĪIJäºĨ +è¾Ľ åĭ¤ +è¶ħ 强 +éĶ ļ +åīįä¸ī åŃ£åº¦ +å°± è§īå¾Ĺ +å´ĩ é«ĺ +è¶Ĭ ä¾Ĩ +è¶Ĭä¾Ĩ è¶Ĭ +å¸Ĥåľº èIJ¥éĶĢ +综åIJĪ ç´łè´¨ +åŃ ļ +ä¾® è¾± +äºĮ åŃĹ +å·¥ä½ľ ä»»åĬ¡ +åı²ä¸Ĭ æľĢ +æľĢ ä¼ĺ +åIJ© åĴIJ +表 çĻ½ +èİ« åIJį +èİ«åIJį åħ¶ +èİ«åIJįåħ¶ å¦Ļ +å¹ £ +åIJĮå¿Ĺ 们 +建设 çĶ¨åľ° +åĦ Ģ +éħį åģ¶ +å¼ © +åĶ± çīĩ +æīĭ èĦļ +åħ¼ ä»» +åģľ æĶ¾ +æŃ£ å®Ĺ +æĸ° åĨľæĿij +åĤ¬ çĶŁ +æīĢ åŃ¦æł¡ +念 ä½Ľ +åĶ¤ éĨĴ +åħ± åĪĽ +æĭī ä¸ģ +èĥĮ çĿĢ +çĶŁæĢģ ä¿ĿæĬ¤ +åı£ 头 +æĸ¹åIJij çĽĺ +調 æķ´ +æĭĽèģĺ ä¿¡æģ¯ +åħ¶ä»ĸ åĽ½å®¶ +ç®Ģ æĺĵ +åĮ¿ åIJį +è¯Ħ æµĭ +æĺ¯ä¸Ģ 座 +çīµ æīĭ +足 迹 +çIJĨ解 åĴĮ +æľĢ åıĹ +å¿ĥ è·³ +çĪ¶ 親 +éĿŀ常 åĸľæ¬¢ +èĭ¦ éļ¾ +æĬĢ å¸Ī +æ°ij æĦı +æĪĺ åĽ½ +æĽ¿ è¡¥ +æ´¥ è´´ +ä¸ŃåĽ½ ä¼łç»Ł +åIJĦ è¡Į +åIJĦè¡Į åIJĦ +åIJĦè¡ĮåIJĦ ä¸ļ +第äºĶ å±Ĭ +èį· èĬ± +æĦı èŃĺ +票 ä»· +åĪĨ æµģ +æĿİ çĻ½ +æ±Ł åĮĹ +æİĴ æĸ¥ +ä½ĵ éĩı +åĮħåIJ« äºĨ +åĪĺ æŁIJ +çİ° å¦Ĥä»Ĭ +å·¥èīº åĵģ +è¿Ļç§į æĸ¹æ³ķ +åĬŀåħ¬ 楼 +çĶµ å·¥ +çħ Ļ +åį¡ çīĩ +å¹´ å¹´åºķ +ä¸ĵ项 èµĦéĩij +åĮ» ç§ij +åĮ»ç§ij 大åѦ +åĽŀ头 çľĭ +ä¸į å±ij +èĩª 驾 +没 æĶ¶ +æīĵ çĮİ +èĦ¸ éĥ¨ +åıĥ èĢĥ +å°Ĩ 士 +è´«åĽ° 人åı£ +çIJĨæĥ³ 信念 +é£İ å°ļ +人æīį éĺŁä¼į +çij ¾ +æĿ¥ è¿ĻéĩĮ +æ´Ĺ 涤 +å¹´ èĸª +èĭį çĻ½ +ä¸ĩ äºĭ +课 æľ¬ +åºĵ éĩĮ +çī¹ æ´¾ +çī¹æ´¾ åijĺ +èµŀ ç¾İ +ç©¿ æĪ´ +製 ä½ľ +èµŀ æĪIJ +ä¸Ģ 侧 +å½ĵåľ° 人 +æĭ İ +纸 è´¨ +ä½Ļ 个 +éĶĤ çĶµæ±ł +æľº åŀĭ +éĻ¢ éĻ¢å£« +åģļ å·¥ +å¼ł è´´ +ç¥Ľ æĸij +æ®ĸ æ°ij +å¥ij 约 +æ¹ĺ æ½Ń +æIJ ĸ +åŃĺ è´§ +交éĢļ 大åѦ +è¶ģ çĿĢ +æĸĩçī© ä¿ĿæĬ¤ +å¤ĩ æĪĺ +éĩĩ 纳 +åįĬ æľĪ +æľĢ åħ³éĶ® +æľĢåħ³éĶ® çļĦ +æİ¥ éĢģ +æĶ¶ åī² +åıį åĢĴ +çĥ Ľ +æ ½Ķ +ä¼Łå¤§ å¤įåħ´ +çļĦè¯Ŀ è¯Ń +容 å¿į +å®ļ éĩı +æķ Ĺ +åĵģçīĮ 形象 +æīŃ è½¬ +åĽ½å®¶ éĩįçĤ¹ +èĨĿ çĽĸ +ä¸Ģ 楼 +大 éĻ¸ +éĤª æģ¶ +åĽŀ åij³ +çĮ ¿ +çĿ¡ åīį +æĹł è¾ľ +çĹħæ¯Ĵ æĦŁæŁĵ +æľºæ¢° åĮĸ +çĤ¹ 亮 +溶 解 +åĩłä¹İ æīĢæľī +è·ij éģĵ +çĶµè§Ĩ æľº +åı ¨ +æijĩ äºĨ +æijĩäºĨ æijĩ头 +èĩª è´Ł +综åIJĪ åĪ©çĶ¨ +èĩª å¦Ĥ +åİŁ ä¾Ĩ +ä¹Łä¸į æĥ³ +èĬĤ 课 +è¿ĩ åī© +çĶ² çĬ¶ +çĶ²çĬ¶ èħº +æĸ° ä¸ĸ纪 +èĩªä¸» åĵģçīĮ +é«ĺ å±Ĥ次 +ä¸Ģ è§Ĵ +è¡Į äºĭ +ç¥ĸ åħĪ +å©ļ åIJİ +éĹ´ éļĻ +ç¼Ŀ éļĻ +è¿Ļ æĶ¯ +ä¸įæĸŃ åĪĽæĸ° +å¾® åŀĭ +æĽĻ åħī +享 çĶ¨ +ä¸ŃåĽ½ 移åĬ¨ +éĹŃ çݯ +æī§ æĦı +åıijå±ķ æł¼å±Ģ +æł¸å¿ĥ åĮº +éªļ æī° +åħļåĴĮ åĽ½å®¶ +ä¸ŃåĽ½ æĶ¿åºľ +帶 èijĹ +ä¸ĩåįĥ çĵ¦ +åħ© 人 +äºİæĺ¯ æĪij +åĽº ä½ĵ +çªģ å¦Ĥ +çªģå¦Ĥ åħ¶ +çªģå¦Ĥåħ¶ æĿ¥ +éĩĮç¨ĭ ç¢ij +çĪ± ç¾İ +æŁ¥ éªĮ +åıĮ èµ¢ +éĹª åħī +楼 å®ĩ +æĻ ı +æľī è¶³å¤ŁçļĦ +æŁĶ æĢ§ +ä¿¡æģ¯ å®īåħ¨ +管 线 +并 ä¸įä¼ļ +åĻ¨ 件 +ä½ł åºĶ该 +çĿĢ å®ŀ +æĺİ æ¸ħ +æĬĹ çĶŁç´ł +æīĵ æŃ» +å®Įåħ¨ ä¸įåIJĮ +èĬ± æ¤Ĵ +æĶ¾ 宽 +ä½İ 端 +åĽĽ èĤ¢ +åĮĹ京 èµĽè½¦ +éĽĨ å¸Ĥ +æľª å©ļ +大å¹ħ æıIJåįĩ +建çŃij 设计 +çĭ¬ æľīçļĦ +æİ¢ éĻ© +æ²³æµģ åŁŁ +æħķ 容 +被 çĽĹ +åĵº ä¹³ +èı ģ +æĥ¬ æĦı +è¶ĬæĿ¥è¶Ĭ 好 +广大 群ä¼Ĺ +å¾· èĤ² +å¸Ĥåľº ä»·æł¼ +奥 å·´ +奥巴 马 +èĬĤ缮 ä¸Ń +两 款 +ä¸ĩä½Ļ åħĥ +ç»´ å°Ķ +çĶŁçī© ç§ijæĬĢ +åIJ¬ èµ·æĿ¥ +çł ļ +æĭŁ å®ļ +æ²¹ çĶ° +声 èªī +建çŃij ä¸ļ +éĻIJ è´Ń +çīĩ åŃIJ +çķľ 禽 +ç½ij é¦ĸ页 +ä¼Ĺ çŃ¹ +æĴŀ åĩ» +åīį ä¸įä¹ħ +åīį ä¸ĸ +åĽĽä¸ª æĦıè¯Ĩ +æµĭ ç»ĺ +éĺ² ç©º +漫éķ¿ çļĦ +æ²IJ æµ´ +æ¯Ķè¾ĥ ç®Ģåįķ +æµĭ å®ļ +åĽŀ è°ĥ +让 人们 +èĴĭ ä»ĭ +èĴĭä»ĭ çŁ³ +ç»ĵ æĻ¶ +å¢ŀæ·» äºĨ +æĿ¡ è¯Ħ论 +åī¯ ä¼ļéķ¿ +ä½ı æīĢ +ç»Ļ åĩºäºĨ +è°ĥ éħį +æ² ĸ +æľī çĶ¨ +æľīçĶ¨ çļĦ +ä¸ĢæĿ¡ é¾Ļ +éĩİ å¤ĸ +ç¼ĺ åĪĨ +æ°¸è¿ľ ä¸įä¼ļ +æŀľ æłij +大åıij å¿«ä¸ī +麻 éĨī +äºij éĽĨ +åİ» åĵªéĩĮ +åħ¥ å¸Ĥ +ä»» æĢ§ +建 æ¡£ +建档 ç«ĭ +建档ç«ĭ åį¡ +ä¸Ģ 棵 +社 åįĢ +缸 ä¼´ +åļ · +å¡« åħħ +ä¸Ģ æĹı +ç¾ ģ +åıĸ è¯ģ +èĪ° éĺŁ +åİĤ åĮº +è¡· å¿ĥ +åıijå±ķ éĺ¶æ®µ +é«ĺ 强度 +åĹĵ åŃIJ +é¢Ĩ è¡Ķ +楼 主 +大 èĴľ +æŀķ 头 +ç²® æ²¹ +é»Ħ çĵľ +æĵ Ĵ +å°ı çĭĹ +æĶ¹éĿ© å§Ķ +åįģ åĪĨéĴŁ +é²ľ èī³ +åħ³ ç¾½ +çĭĢ æħĭ +å®ŀçĶ¨ æĢ§ +å°ij è§ģ +é£ŀ æī¬ +çĶ° éĩİ +æIJ Ĥ +è¿Ļ个 è¯į +åºĶæĢ¥ é¢Ħæ¡Ī +è§Ĵ度 æĿ¥çľĭ +æķ¬ çķı +æ³ķ å®Ŀ +åĸĦ æĦı +æīĵ æĸŃ +对 åĨ³ +çµķ å°į +åĢŁ æѤ +å¼Ģ æºIJ +å°ı 說 +ç¥ º +å²ģ 以ä¸ĭ +éĢĢå½¹ åĨĽäºº +ä¸įä¹ħ åīį +åĩº åİĤ +讽 åĪº +æĿ¥çľĭçľĭ åIJ§ +éŃĶ åħ½ +çķĻ ä¸ĭæĿ¥ +å±ħ 室 +åłħ æĮģ +çľĭ äºĨä¸Ģ +çľĭäºĨä¸Ģ çľ¼ +éĽĨåĽ¢ æĹĹä¸ĭ +æĪĺ æĪĺç»ĦåIJĪ +è®¤çľŁ èIJ½å®ŀ +汽车 产ä¸ļ +çī©çIJĨ åѦ +æķ µ +éĴ Ŀ +åĽ¢ éķ¿ +ä¸įæĸŃ æī©å¤§ +èĤ© è´Ł +åıijå±ķ 缮æłĩ +è³ĩ éĩij +åīį ç½® +ä¸ŃåĽ½ åı¤ä»£ +æŃ» åĪij +åħħåĪĨ ä½ĵçİ° +åħ³ éŨ +ç¾İ æĦŁ +æīĵ åħ¥ +æĬijéĥģ çĹĩ +å°ij çĪ· +æłij æŀĿ +æ¶Īæģ¯ 称 +æ´Ľ åħĭ +åį ¯ +è¿Ī åIJij +æݨ åĭķ +ä»İä¸ļ èĢħ +åİ» ä¹° +欢 å¿« +æĭ¥ æĮ¤ +马 桶 +æĬĬ æݧ +æĶ¿ åħļ +å¼ł æī¬ +客 æłĪ +红 æĺŁ +éĢģ æĿ¥ +åħ¨åŁŁ æĹħ游 +èĩª ç§ģ +åįģäºĮ æĿ¡ +åı¹ æģ¯ +ä¸Ģ èīĺ +ä¿Ŀ è´¹ +æĸ½å·¥ çİ°åľº +æľī 幸 +ç»Ń èĪª +åı¯èĥ½ æľĥ +èĥĮ åıĽ +ä½£ éĩij +ä¸ī çŃīå¥ĸ +å¾Ī 满æĦı +游æĪı åī¯æľ¬ +群 éĩĮ +æŀĦ 件 +åºı å¹ķ +太 æ¹ĸ +æľ¨ è´¨ +æĻĭ æ±Ł +çµĤ æĸ¼ +è·³ è·ĥ +åĢºæĿĥ 人 +çŃī 诸å¤ļ +æĶ¾ åĩº +åħ³éĶ® æŶåĪ» +æĦŁæŁĵ èĢħ +é£ŀè¡Į åijĺ +èĥĨ åĽº +èĥĨåĽº éĨĩ +æĬ± æŃī +åij¨ äºĮ +æĸ° æĹ¶æľŁ +åĨ·éĵ¾ çī©æµģ +è¿Ļç§į æĸ¹å¼ı +该 æĿij +åĽŀ é¦Ī +åŁºçĿ£ æķĻ +人 åıĤ +æŀ¯ çĩ¥ +æī¹åıij å¸Ĥåľº +åħħåĪĨ èĤ¯å®ļ +å¸Ĥ æĶ¿åįı +äºĭ æ¥Ń +龸 çİĭ +çĥŃ æIJľ +åįģä¹Ŀ 大 +ä¼´ æľī +ç¾İåĽ½ æĢ»ç»Ł +åŁİå¸Ĥ 管çIJĨ +ä¸ĭ 令 +èĥ¸ åı£ +åıª çŁ¥éģĵ +åij¨ ä¸ī +çĶ¨ æĪ¶ +éŃ ¯ +å¿ĥ è¡Ģ +带头 人 +åĮ» åĬ¡ +åĮ»åĬ¡ 人åijĺ +æݧåĪ¶ åĻ¨ +ä½ľåĵģ åĨħ容 +æĪĺ åıĭ +åİĨ å¹´ +ä¸į åħĭ +ä¸įåħĭ ä¸įåıĬ +æĹ¥ æŃ£å¼ı +è±IJ å¯Į +ç¨İ è´¹ +æŶ æķĪ +å±ķ ä½į +è¡¡ éĺ³ +æĪ¿ 貸 +çĪĨ 款 +ä¹IJ æĦı +çĶ· 主 +å¯ ¬ +æľĥ èŃ° +ä¹ĭ å¤ľ +åIJĮ 樣 +ä¸įè¦ģ 太 +ä¼Ĭ æĸ¯ +ä¼Ĭæĸ¯ åħ° +åŁºæľ¬ åİŁåĪĻ +åİ» æİī +ä½İ ä¿Ŀ +个 交æĺĵ +个交æĺĵ æĹ¥ +èģĬ èģĬ +åĽĽ ä½į +åħļç»Ħ æĪIJåijĺ +主è¦ģ ä»İäºĭ +å½± éŁ³ +åĨĴ åĩº +åij¼åIJ¸ éģĵ +è¾¾ å°Ķ +æľ¨ åľ°æĿ¿ +诡 å¼Ĥ +çģ¯ åħ· +çģ« çĥ§ +解 èĦ± +æĦĪ åıij +æ¹ĸ å·ŀ +é£İ ä¿Ĺ +æĸ° å½¢åĬ¿ +æĸ°å½¢åĬ¿ ä¸ĭ +è² Ŀ +èĦ ĵ +åĬ¨åĬĽ çĶµæ±ł +é£ŀ èĪ¹ +飧 æĢ§ +åĪ© çī© +åĪ©çī© 浦 +ä¸į 认è¯Ĩ +ç¼ĸ ç»ĩ +ä½ľ åĿĬ +èģĮä¸ļ æĬĢèĥ½ +çľĭ è¦ĭ +åĽ´ æ£ĭ +æĺı è¿· +å½Ĵ å±ŀäºİ +æĤ¬ å´ĸ +éĨ« çĻĤ +å®ĭ 代 +åºĦ æĿij +èĹ ķ +çĮĽ çĦ¶ +çĩĥæĸĻ çĶµæ±ł +å®ŀä½ĵ åºĹ +ä¸į足 以 +æĥħ ç· +æĥħç· Ĵ +å»Ĭ åĿĬ +çĶµ åı° +åºĶ åĬĽ +ä¸Ńå°ı åѦçĶŁ +èĥ¡ åIJĮ +éī´ åĪ« +åĨħ ç½® +ä¹± 象 +æ¬Ĭ çĽĬ +å¼ĢæĶ¾ å¼ı +åįļ æĸĩ +讲 课 +çŃī åİŁåĽł +ç©· 人 +交 æĽ¿ +æĬ¤ çħ§ +åıijå±ķ æľºéģĩ +客 åķĨ +åıį ä¹ĭ +ç±³ é¥Ń +并 åıij +并åıij çĹĩ +æ±ī åŃIJ +æŀľ åĽŃ +对æĪij æĿ¥è¯´ +åģı åIJij +æī¹ 示 +读 åIJİ +读åIJİ æĦŁ +æĺİ æĻº +åĽ´ çĿĢ +åıį 转 +æĿ¨ å¹Ĥ +ä¸ĵ åįĸ +ä¸ĵåįĸ åºĹ +åıĹ éĻIJ +åºŁ è¯Ŀ +æŀģ å°ij +åįĪ åIJİ +è¿Ľ ä¿® +åīĬ åĩı +æľ¬ç§ij çĶŁ +ä¼ĺ éĢī +åħī çħ§ +åıĻ äºĭ +åıĸ æļĸ +åĮĹ è·¯ +æ¦ ķ +èİĨ çĶ° +楼 å±Ĥ +天 èĬ± +天èĬ± æĿ¿ +çĤ ľ +å·²ç»ı æľīäºĨ +è¶ ¾ +çĶ³ åįļ +çĶµ éĺ» +åĬŁ 课 +æŃ¥ æŃ¥ +éĤ£ä¹Ī 容æĺĵ +æѤ æĸĩ +ä½ ° +计 è¾ĥ +çīĩ éĿ¢ +çĶµå½± éĻ¢ +ä¸į åħ¬å¹³ +ä¸ī æľŁ +æĹħ游 èµĦæºIJ +å¤ļç§į å½¢å¼ı +è£Ĥ ç¼Ŀ +åIJİ æİĴ +硬 度 +åĽŀ æļĸ +éģĵ æķĻ +è´« è¡Ģ +æ¸ħ é¦Ļ +伤 çĹħ +æĦı 義 +çļĦ ç¼ĺ +çļĦç¼ĺ æķħ +åºĦ 严 +åıªæĺ¯ ä¸ºäºĨ +æīĵ æĬĺ +以 ä¾Ĩ +滿 足 +çİĽ 丽 +風 éļª +æĸĩ ç§ij +éħįå¤ĩ äºĨ +è¿Ľ é£Ł +æ¶ ¡ +è·¯ ç¨ĭ +åı« 声 +ä¸Ńå¿ĥ åŁİåĮº +æľīæīĢ ä¸įåIJĮ +å¼µ è²¼ +é¢Ħ æĬ¥ +æľīå¤ļ ä¹Ī +è¿Ľè¡Į åħ¨éĿ¢ +æĽ¾ ç¶ĵ +ä¸ī 代 +å®ı 大 +æ¸ħ æī« +éĢī åĩº +åĵª ä¸Ģ个 +主 義 +ä¾Ŀ æĵļ +çļ® éĿ© +赶 æĿ¥ +çŃĽ æŁ¥ +æ¨ Ł +ä¿Ŀ èįIJ +åIJĥ æĥĬ +æľĭåıĭ们 对 +ä»ĸ æĺ¯ä¸Ģ个 +åºŁ æ°Ķ +æ» ħ +è´¢ ç¨İ +æĿij æĿijæ°ij +èµĦ产 è´ŁåĢº +å®ī å¨ľ +缮åīį åĽ½åĨħ +æĦŁè§ī èĩªå·± +çµIJ åIJĪ +éĶ¦ æłĩ +éĶ¦æłĩ èµĽ +æĽ´ æ·± +åŁº æķ° +éħ¿ éħĴ +çī¹èī² 产ä¸ļ +åİĭ å®ŀ +ä¾Ŀæ³ķ 追究 +æ·¡ å®ļ +ç®Ģ缴 å°±æĺ¯ +å£ĵ åĬĽ +æ°ij å¿ĥ +ä¸į åIJĪéĢĤ +çĶ±æѤ åı¯è§ģ +èµŀ èªī +æ¾ ¤ +åĩłå¹´ åīį +åIJī ä»ĸ +çł´ æįŁ +轻轻 åľ° +å²Ľ 屿 +æĦı å¢ĥ +ä»Ģä¹Ī åı« +åģĩ è£ħ +éĢģ è´§ +å¹ķ å¢Ļ +妥 åįı +åĽ½ æĹĹ +äºĨ å¾Īä¹ħ +åĪĨ辨 çİĩ +ç´ Ķ +éĺ³ åĮº +åĩŃ çĿĢ +åģľè½¦ ä½į +京 éĥ½ +éĶ £ +æĵ ¾ +è¿Ľ éŨ +åĪĺ æµ· +åĽĽ 级 +女 足 +è¡ĮæĶ¿ 审æī¹ +éģ¥ æݧ +ä¸į éĮ¯ +å¾Ĺ å¾Ī好 +为 缮çļĦ +ä»į æľª +ç²¾ è£ħ +éĢį éģ¥ +å°½ 头 +çºł ç¼ł +éłĺ å°İ +æĭħ è´Ł +æĪĸèĢħ åħ¶ä»ĸ +åıªä¸įè¿ĩ æĺ¯ +åı® åĺ± +åģĩ åĨĴ +æļĸ æ°Ķ +çĽIJ åŁİ +被 è§Ĩ为 +诺 è´Ŀå°Ķ +ç»ĻäºĨ æĪij +è¿ij åįĥ +éĩį åĽŀ +éĨĴ äºĨ +çĶµ 解 +忽çķ¥ äºĨ +èĥĮ éĥ¨ +æĸĩæĺİ åŁİå¸Ĥ +æº ħ +è² ĵ +æĬµ æĮ¡ +åĸľæ¬¢ åIJĥ +éĿĻéĿĻ åľ° +å¾Ī æ·± +åŁºç¡Ģ çŁ¥è¯Ĩ +è¿ĩ éĶĻ +çIJĨ ç§ij +交æµģ åIJĪä½ľ +èĪ Ķ +調 æŁ¥ +æħĪ æĤ² +éĴ ° +èĩ´ çĶµ +å®£ä¼ł æ´»åĬ¨ +åıĺ éĩı +çļĦ人 æĿ¥è¯´ +æŶ éļĶ +ä¸į管 ä½ł +缸 è¿ij +è´µ éĩijå±ŀ +ä¹Łä¸į åı¯èĥ½ +ç²ī æľ« +åįĹ çĵľ +çĻ½ 马 +åħī æºIJ +éĩij å¥ĸ +çĭ¬ è§Ĵ +çĭ¬è§Ĵ åħ½ +妨 ç¢į +ç»Ļ åĬĽ +ä½Ĩ ä»į +å¼łå®¶ åı£ +èIJ¬ åħĥ +渲 æŁĵ +éķ¿å¤§ äºĨ +è®°èĢħ äºĨ解 +æĢĢ çĿĢ +è¦ģ åѦä¼ļ +游æĪı 代 +游æĪı代 ç»ĥ +äºĮ çĻ¾ +æĦıè¯Ĩ å½¢æĢģ +çİ º +计åĪĴ çĶŁèĤ² +æī¾ åĩĨ +åħ° èĬ± +è¿Ļ座 åŁİå¸Ĥ +污 æ³¥ +å®ĺæĸ¹ 微信 +å½Ĵ å±ŀ +æ°§ æ°Ķ +éģİç¨ĭ ä¸Ń +åį°è±¡ æ·±åĪ» +稳 妥 +çµIJ æĿŁ +åŃķ æľŁ +çī¹ æĿĥ +åĿļ åĽº +顺 åĬ¿ +æŀľ èĶ¬ +éĨ« 師 +åİ ® +ä¹Łæĺ¯ å¦ĤæѤ +é¦Ĵ 头 +缸 åĬ© +å¹² 线 +ä¸Ģ æľ¬ä¹¦ +ç» ¥ +æĮ¯ å¥ĭ +èĤ¾ èĦı +åĭķ çī© +é£ŀ è·ĥ +èıľ åĵģ +å¤ļ ä½Ļ +å¤ļä½Ļ çļĦ +éĢĿ ä¸ĸ +æģĭ 人 +å¼Ģåıij åĪ©çĶ¨ +顺 丰 +éĩİ å¿ĥ +æł¡ å¤ĸ +æģIJ é¾Ļ +éĿ¢ åħ· +éķ¿ è¾Ī +éļı å¤Ħ +éļıå¤Ħ åı¯è§ģ +ç´§ 缺 +éĩį ä¸Ń +éĩįä¸Ń ä¹ĭ +éĩįä¸Ńä¹ĭ éĩį +奥 æĸ¯ +奥æĸ¯ åį¡ +ä¸Ģ个 å¤ļ +ä¸Ģ个å¤ļ æľĪ +ä¸įåı¯ 缺å°ij +æĸ° æł¼å±Ģ +æıIJ æĮ¯ +è¡Į è´¿ +æ¼Ĥ æµģ +èģĬ åŁİ +åħ´ 建 +è´¨ æ£Ģ +ç§ģæľį 游æĪı +æĽ´ éĩįè¦ģ +è´ ® +çħ ľ +转åıĺ 为 +è¿Ļ 两年 +ä¿Ŀ é²ľ +æī§ æķĻ +çĥ ¨ +å¼Ģåıij 建设 +è¿IJèIJ¥ 管çIJĨ +误 å·® +京 åī§ +å¸IJ åı· +å·¥ä½ľ ä½ľé£İ +ä¸ĸ ä¿Ĺ +çĻ½ 宫 +天 åĽ½ +å¤©åĽ½ 继ç»Ń +å·´ æĸ¯ +èIJ¥ åĪ© +åĵģ æł¼ +æĿijæ°ij 们 +æĪ¿ 车 +çŃī çĹĩçĬ¶ +å¦Ĥ å®ŀ +å® ¸ +å±Ĥ 级 +éĶĻ è¿ĩäºĨ +ç»ĵ å®ŀ +ç¬ij èĦ¸ +羣å®ŀ æĢ§ +éĥ½å¸Ĥ æĬ¥ +é¥Ń èıľ +åºĶ 注æĦı +æĬ½ çĥŁ +伪 éĢł +åīį ä¸Ģ天 +éŃĶ é¾Ļ +éŃĶé¾Ļ 令çīĮ +约 è°Ī +绣çŃ¹ æİ¨è¿Ľ +让 çĶ¨æĪ· +åħ¨éĿ¢ èIJ½å®ŀ +å¼Ħ å¾Ĺ +è°Ī æģĭçĪ± +鸣 æĪIJéķ¿ +鸣æĪIJéķ¿ è®° +æ´ĭ æ´ĭ +çĸı æķ£ +éĿ¢ç§¯ 约 +æµĵ 缩 +æĸ¯ é¡¿ +çĶŁæĢģ åľĪ +æī§ 导 +移 éĢģ +齿 è½® +æł¹æľ¬ å°±ä¸į +缩 åĩı +èµ° ä¸ĭåİ» +çĿ« æ¯Ľ +ä¹Łä¸į éĶĻ +åıįæĺł åĩº +èĭ¦ æģ¼ +缸åħ³ æĶ¿çŃĸ +é«ĺ 楼 +ç²ī èī² +æĬķèµĦ é¢Ŀ +ä¸į ç»ı +ä¸įç»ı æĦı +å®ģ æĦ¿ +èĪĮ 头 +æ»ĭ çĶŁ +å®ģ åİ¿ +åīįåĪĹ èħº +åĩ ³ +é£Ł 欲 +åıĸ èĥľ +éĻ¢ åŃIJ +ç´łè´¨ æķĻèĤ² +滨 å·ŀ +æĬ¢ æĬĵ +å¼Ĥ åij³ +åĴ ļ +åĬ į +宽 éĺĶ +æļ´ 涨 +æĥł åıĬ +è§Ħ ç¨ĭ +ä¾Ľ åħ» +éĢģ å¾Ģ +å±± åºĦ +举 äºļ +å±ķ é¦Ĩ +解 éĶģ +æĹł è§Ĩ +éĻį èIJ½ +è¿ŀ äºij +è¿ŀäºij 港 +åıĤ è°ĭ +çİ ĸ +ç¬ ĥ +èĢĹ è´¹ +æī¿ å¾· +社ä¼ļ æķĪçĽĬ +åįĹæµ· ç½ij +åĪĽ 伤 +èIJ ± +åħħ æ²Ľ +ç½ijç«Ļ 建设 +大 åºĨ +åĨį éĢł +åŃĹ æł· +åħ¨æ°ij åģ¥èº« +èĮ« èĮ« +æµ® åĬ¨ +åīį åı° +å¢ŀ 设 +éĢĽ è¡Ĺ +åĢĴ éĹŃ +æ³ķå¾ĭ 顾éĹ® +çĸ ® +çĹħ çĹĩ +空 åīį +请 æķĻ +èĥľ ä»» +æĿĢ èıĮ +æĪĺæĸĹ æľº +ç»ĺ åĪ¶ +å¤Ħ æĸ¹ +çªģ åĽ´ +çĮ« åĴª +æĬ¥åijĬ æĺ¾ç¤º +ç¿ Ł +çķ¶ åľ° +æľĢ éļ¾ +纪 å§Ķ书记 +ä½İ åİĭ +èĻļ 空 +è¿Ļéĥ¨ çĶµå½± +产ä¸ļ åįĩ级 +è°· çĪ± +è°·çĪ± åĩĮ +æĬ¼ éĩij +女 æĸ¹ +éĴ» çłĶ +æļĹ æļĹ +è¿· ä½ł +æīĢ è¬Ĥ +å¨ģ å»ī +å¼Ģ æľĹ +å² Ķ +çģ« çĤ¬ +åIJĪçIJĨ æĢ§ +åħ¬ åĬŀ +ä¼ļ ä¼ļéķ¿ +éĺ´ è°ĭ +å¼Ģ å±Ģ +æĻ®éĢļ è¯Ŀ +åį¡ æĭī +å°ij åIJĥ +éĹª èĢĢ +æŀľ æ±ģ +æī§è¡Į åĬĽ +è° Ľ +æĬ¢ åĬ« +é«ĺéĢŁ åıijå±ķ +éŁ ¬ +åįĹ æ²Ļ +é«ĺçŃī åŃ¦æł¡ +æį¢ 个 +åı¯èĥ½ åŃĺåľ¨ +æĬ Ĵ +è°± åĨĻ +被 æĬĵ +æĿ¯ åŃIJ +èĬĤèĥ½ åĩıæİĴ +æ°ĶåĢĻ åıĺåĮĸ +åĪĨ åĪ¥ +ä¸Ń æŀ¢ +欢 åij¼ +åħī 纤 +è¿Ļ 群 +çľ¼ çķĮ +åħ±åIJĮ åıijå±ķ +çİ° ä»Ĭ +éĹ» è¨Ģ +çī¹èī² å°ıéķĩ +æķij 人 +éĻį æ°´ +ä¸ĸçķĮ ä¸Ģæµģ +å°± é¤IJ +çŀ ¥ +å¤į ä»ĩ +ç¾½ æ¯Ľ +ç¾½æ¯Ľ çIJĥ +è´© åįĸ +æºIJ æ³ī +æĢ»ä½ĵ è§ĦåĪĴ +åĬ¨ æĦŁ +ä¸Ģ 审 +åĢŁ éĴ± +è§ģ æķĪ +èĬ± èįī +åIJĮ ä¸ļ +æŁ¥ è©¢ +åĽ½éĻħ åIJĪä½ľ +ä¾Ľ åĽ¾ +åģ ´ +æł ĵ +缸 éĢļ +è°Ī åıĬ +è¿ĩç¨ĭ å½ĵä¸Ń +é¦Ļ èıĩ +åįģåĽĽ æĿ¡ +ä¸Ģå¼Ģå§ĭ å°± +ä¸ĵ åijĺ +æĺİ é¡¯ +æīĵéĢł åĩº +ä¸ĭéĿ¢ æĪij们 +æľº æ²¹ +åı° è¯į +åŃIJ å¼Ł +æľĢ 常è§ģçļĦ +æĪij è®°å¾Ĺ +ç» ° +æĤ¬ æµ® +è¿ĺ 羣æĺ¯ +æĮĤ åı· +åıĭ åĸĦ +éĩį 伤 +çħ§ 亮 +æѦ èѦ +åĩºçİ° éĹ®é¢ĺ +è¸Ĭ è·ĥ +åľ°çIJĥ ä¸Ĭ +å¸Ĥ 人大 +åıĹ害 人 +å² IJ +åIJĮ åѸ +éĩijèŀį å¸Ĥåľº +æľīçļĦ çݩ家 +å¸Ĥ æķĻèĤ² +å¸ĤæķĻèĤ² å±Ģ +åIJĦ å¼Ĥ +ç·ļ ä¸Ĭ +æģ º +æľī 大éĩıçļĦ +åķĨ æĬ¥ +åįķ åįķ +åħ¨ é¢Ŀ +ä¾Ŀæŧ æĺ¯ +好 åĩłä¸ª +åĸ µ +éĩį æķ´ +çĶŁæ´» è´¨éĩı +æİ¢ 访 +åį° èĬ± +缼 è¡Į +å¾® è§Ĥ +èĪį å¾Ĺ +åºŁå¼ĥ çī© +积 èĵĦ +å®ļ å±ħ +æĤ ¼ +èĮ ¸ +çļĦ 帮åĬ© +çļĦ帮åĬ© ä¸ĭ +亿 åIJ¨ +åŃĶ éĽĢ +è¿ĻæĿ¡ è·¯ +é¥ µ +æĦĪ åĬł +éķ į +ä½ľ æ¡Ī +èįĶ æŀĿ +太 å°ij +è·» 身 +åħ¬çĽĬ æ´»åĬ¨ +çĻ½ æĸij +æĬĢæľ¯ æ°´å¹³ +å¸ § +æĹł çŁ¥ +åºĶ该 æĢİä¹Ī +éĢĢ å¸Ĥ +æ¸ Ń +åħ» çĮª +é© ¼ +群 å²Ľ +大 åį« +ä¹ĺ çĶ¨è½¦ +èı² å°Ķ +è´´ åIJ§ +åģľ ä¸ĭæĿ¥ +æľīæľº ç»ĵåIJĪ +åĪ» èĭ¦ +çļĦ åľ° +çļĦåľ° æŃ¥ +è¯Ĭ æīĢ +å¼Ģ æĪĺ +èĢģ çīĮ +çŃ¹ çłģ +åħ«å¤§ 以æĿ¥ +楼 æĪ¿ +åŃĻ æĤŁ +åŃĻæĤŁ 空 +åħĴ åŃIJ +第ä¸Ģ æĿ¡ +社交 åªĴä½ĵ +æĥ³ èµ·æĿ¥ +大 æ´ĭ +æĭ¼ éŁ³ +è¿Ľ åįļä¼ļ +è¿ĩ åħ³ +æ² ¼ +ç©¿ æIJŃ +éĤ£ ä¸Ģ天 +çł´ éŨ +æĬķæłĩ 人 +èµ¢ 家 +èĻļ å¼± +æ¿ ĥ +å®ī æ£Ģ +客 家 +çĭ¬ç«ĭ èij£äºĭ +æīĭ åĬ¿ +åīµ éĢł +åľĨ满 å®ĮæĪIJ +为主 线 +好å¥ĩ å¿ĥ +é¢Ĩ åľŁ +çª ĸ +åħ¸åŀĭ æ¡Īä¾ĭ +çªģåıij äºĭ件 +åºķ æ°Ķ +头 æĻķ +å®Ľ å¦Ĥ +è§ ¸ +æ¸ħ æ·¡ +åļ ¼ +åģľ çĶµ +ç²ī å°ĺ +éĻįä½İ æĪIJæľ¬ +æĶ¾ æīĭ +è®°èĢħ 表示 +æĭĸ 延 +éª ĩ +æ®ĭ å¿į +çľģ æķĻèĤ² +çľģæķĻèĤ² åİħ +é«ĺ é¢Ŀ +éĦ Ļ +æ¥ ŀ +åĨħ ç§ij +èIJ¥ä¸ļ é¢Ŀ +åŁº çŁ³ +æµģ æ·Į +主 æŨ +éĺIJ éĩĬ +建 åįİ +æĥĬ åı¹ +çī¢åĽº æłijç«ĭ +æĺ¯åIJ¦ åŃĺåľ¨ +建 åĨĽ +éĽ¾ éľ¾ +åħ¬ 认 +åħ¬è®¤ çļĦ +æ°¨ åŁº +æ°¨åŁº éħ¸ +åīį åĩłå¹´ +åĪ¹ éĤ£ +æ±Ł 举 +å·¥ æ¥Ń +ä¸ĢçĤ¹ ä¹Łä¸į +ä¿® 士 +äºĨä¸Ģ éģį +åĪ ģ +æ»ļ æ»ļ +åĪĨ æł¡ +羣 çĪ± +è¡Ģ èĦī +æĢ¥ åī§ +ä¸Ģ群 人 +ç¾ ¯ +æĪIJ é¾Ļ +ç²¾ç¥ŀ çĹħ +缸åħ³ 人åijĺ +éĿĵ 丽 +ä¸ī åŃ£åº¦ +åĪĴ å®ļ +ä¸ĸçķĮ 第ä¸Ģ +éĢļ ä¿Ĺ +åķĨä¸ļ åľ°äº§ +åĬŁèĥ½ æĢ§ +èµĦæľ¬ 主ä¹ī +详 è§ģ +æĬĵ æįķ +æĸĩ æĺĮ +å®Ŀ å®ī +è£ħéħį å¼ı +æºIJ æºIJ +æºIJæºIJ ä¸įæĸŃ +çĶŁ æĢķ +纵 åIJij +å£ ½ +çľ¼ è¢ĭ +èĤī ä½ĵ +åı¤ ä»Ĭ +èŀį åªĴä½ĵ +åģ ī +æł¼ æľĥåĵ¡ +çĥ · +åĬŁ çĶ¨ +æīŃ çŁ© +绿èī² éĢļéģĵ +åī§ ç»Ħ +å¼± åĬ¿ +è´¨éĩı éĹ®é¢ĺ +éĻIJ é¢Ŀ +éª Ĩ +éģµ ä¹ī +å¯Ŀ 室 +æĥ³ 念 +åł± åijĬ +ä»ħ 次 +ä»ħ次 äºİ +èŀį åĪĽ +æĭĽèģĺ ä¼ļ +åºĬ åŀ« +转åŀĭ åıijå±ķ +ä¸ŃåĽ½ çĶµä¿¡ +åIJ¬ è¯Ŀ +è«ĭ æ±Ĥ +大éĥ¨åĪĨ 人 +æ´» å¾Ĺ +åĵŃ æ³£ +è¶ Ļ +åıijçĹħ çİĩ +ä¸į 符 +åĨĽ å®ĺ +é¢Ī æ¤İ +æĸ°åĨł çĸ«æĥħ +æŁ¬ åŁĶ +æŁ¬åŁĶ 寨 +ä»»ä½ķ å½¢å¼ı +人 éĻħ +人éĻħ åħ³ç³» +æĢ» æī¿åĮħ +å¹³åĿĩ æ¯ı +æģŃ åĸľ +åĦ ĺ +åħµ 马 +è¿Ł åĪ° +å·¥ 伤 +çīĪæĿĥ å½Ĵ +çīĪæĿĥå½Ĵ åİŁ +æĭ¥ æĬ¤ +ç³Ĭ æ¶Ĥ +å¹² æ¶ī +å°ij ä¸įäºĨ +æĥ³ æī¾ +è´¹ çİĩ +该 éĻ¢ +èŀį åĮĸ +è¿İ åIJĪ +è§ĨåIJ¬ èĬĤ缮 +æł¼ 網ç«Ļ +çľī æ¯Ľ +欢è¿İ 大家 +家åºŃ æķĻèĤ² +ä¾µ èļĢ +ç»Ļ ä½łä»¬ +è¡Ģ液 循çݯ +å¯Ħ æīĺ +å°ĸ åı« +以ä¸ĭ åĩłä¸ª +è¿ĺ 以为 +åħ¶ä»ĸ çݩ家 +ç¬ij ç¬ij +æīĵ åIJ¬ +èĩªçĦ¶ ç§ijåѦ +åŁº ç«Ļ +ä¹Ŀ å·ŀ +ä¿Ŀ 驾 +ä¿Ŀ驾 æĬ¤ +ä¿Ŀ驾æĬ¤ èĪª +æĶ¾ çľ¼ +çŁ¥åIJį ä¼ģä¸ļ +ç¸ ® +ç¨ ½ +æļ ĩ +使çĶ¨ 網路 +é¢Ħ çķĻ +大 象 +åıijæĺİ ä¸ĵåĪ© +æĸĩ 娱 +éĢł ç¦ı +湿 润 +éĿ¢ æĿ¡ +æ¶Īè´¹ åįĩ级 +è®Ĭ å¾Ĺ +åĩł åIJį +ä» Ħ +认 æ¸ħ +è¿ľ æĻ¯ +æıĴ 座 +诸 侯 +åıĺ æĢģ +ç¦ı 彩 +è´§ æŀ¶ +失 æݧ +移åĬ¨ 端 +ä¸Ĭ åı¸ +éĢł 纸 +å¸ĥ æľĹ +çĴ ĩ +åı° åįĹ +åĮĹ京 åĨ¬å¥¥ +èĵĿ çīĻ +éķ¿ çŁŃ +æĬĺ å°Ħ +ç»ij æŀ¶ +å¯Ĵ åģĩ +转 åŁºåĽł +æĢ¥ äºİ +æŃ£ åĵģ +åħħ 滿 +大 纲 +æĬĹ ä½ĵ +è¨ĵ ç·´ +æĶ¶ ç´§ +æ¯Ķ è³½ +åħµ åĬĽ +æľ¬ æĽ¸ +äºĮ 代 +æĢ¥ è¯Ĭ +æĸĩ æ¡Ī +ç»ı åķĨ +æĻ¨ æĬ¥ +æ£ ĺ +æĢ»ä¹¦è®° åľ¨ +åıĹ éĤĢ +äºĶ åĽĽ +å²Ń åįĹ +çĪ± åIJĥ +åŁĥ å°Ķ +å¿ĥ å¢ĥ +è¦ĨçĽĸ éĿ¢ +å®ŀåľ¨æĺ¯ å¤ª +æł¹ åºķ +纷纷 表示 +åĹ ħ +éļıçĿĢ æŶéĹ´ +åİĨåı² æĤłä¹ħ +éħ ī +æĢ» éĺŁ +主é¢ĺ æ´»åĬ¨ +éĹ® åį· +é©¿ ç«Ļ +æı¡ ä½ı +åı¯èĥ½ 导èĩ´ +æ°ij éĸĵ +éĸĭ åķŁ +ä½Ĩ ä¸įéĻIJ +ä½Ĩä¸įéĻIJ äºİ +åįģ éĩĮ +å¨ ¥ +æįŁ èĢĹ +çĸı 导 +çݯ æ°§ +ç¥ŀ éĢļ +çĪ± å°Ķ +çĪ±å°Ķ åħ° +æľ´ å®ŀ +å¿« æĬ¥ +æĶ¶ åıĹ +æĪĸ 許 +èĥĮ éĿ¢ +æĸĩåĮĸ ä¼łåªĴ +ä¸ī åĢĭ +æĶ» åĬ¿ +å®ī 举 +å®ī举 å°¼ +åĿĩ å·² +顾 èĻij +éĦ Ń +è¿Ļ家 åħ¬åı¸ +åħ¬åijĬ 称 +æıIJä¾Ľ ä¼ĺè´¨ +稳æŃ¥ æİ¨è¿Ľ +å¤į è¯ķ +å°Ĩ é¢Ĩ +è°Ī èµ· +å¨ Ħ +è¿ŀ 线 +æ©Ł éĹľ +åºĶçĶ¨ åľºæĻ¯ +çĶ» åĥı +è´¢ è¿IJ +ä¿Ŀ éļª +çĹħ çIJĨ +æ¯Ľ 主å¸Ń +ä¸Ŀ 毫ä¸į +çĪ± å¥ĩ +çĪ±å¥ĩ èīº +ä¸ĵ家 ç»Ħ +åij¼ åĶ¤ +éĭ ¼ +çģ ¸ +é¢ĨåħĪ åľ°ä½į +æıIJ æĭĶ +龸 éģĵ +å±± åĿ¡ +èĿ İ +沸 èħ¾ +该 项 +ä»Ĭ çĶŁ +ä¸Ģç¯ĩ æĸĩç«ł +æĸ¹å¼ı è¿Ľè¡Į +é»ij 客 +æĶ¹ åĬ¨ +主 é¡Į +æķ£ å¸ĥ +ä»Ģä¹Ī åľ°æĸ¹ +åĮĸ åIJĪ +åĮĸåIJĪ çī© +éĿĻ çĶµ +æĢ» æĶ¶åħ¥ +å§Ķ ç»Ħç»ĩ +å§Ķç»Ħç»ĩ éĥ¨ +éĿĻ æĢģ +èĢģ åŃĹåı· +室 åıĭ +éĥ½ä¸į æķ¢ +æŀ¶ åŃIJ +çģµ æķı +审 è§Ĩ +æĤ£ åĦ¿ +å±± 寨 +èĸª èµĦ +é©° æı´ +éĥ¨åĪĨ åĨħ容 +好 ä¼¼ +æĪIJåijĺ åĽ½ +åľ¨æĪij çľĭæĿ¥ +åħ³æ³¨ 度 +éĻĪ æŁIJ +è¿Ļç§į äºĭæĥħ +éĢī å®ļ +ç²¾ åŃIJ +å£ģ çĶ» +æ±Ł æ·® +é«ĺ æĺĤ +æł¼ åĬĽ +è¼ © +åѦ åłĤ +æĤ¨ åIJĮæĦı +ä¸ĢåĪĩ éĥ½æĺ¯ +æ½ ¤ +éĸ ĥ +å¸ĮæľĽ èĩªå·± +ä¿ ĺ +æ±Ł åİ¿ +æ³ ¾ +ç§ij æķĻ +æīĵ è¿Ľ +ä¸į æħİ +å¯Ĵ åĨ¬ +æ¸Ķ æ°ij +鼷 æĸ¯ +主 å®° +æĹħ游 度åģĩ +çĶµåŃIJ éĤ®ä»¶ +æ±Ĥ å©ļ +éļİ æ®µ +åģ¥èº« æĪ¿ +注æĺİ åĩºå¤Ħ +äºĭæķħ åıijçĶŁ +级 以ä¸Ĭ +åŃĺ æ´» +æĸ½ èĤ¥ +èľľ èľĤ +åµ © +æĮĸæİĺ æľº +æĬĹ æĭĴ +ä¼ł 导 +æĺ¯ä»Ģä¹Ī åij¢ +ä¸Ĭå¹´ åIJĮæľŁ +建 åħļ +çĶŁ æħĭ +ä¿Ŀ ä½ı +款 车åŀĭ +人 èĦī +éļIJ èĶ½ +失 æķĪ +éģ¿ åŃķ +ç®Ģ 便 +谢谢 ä½ł +å®Ī ä½ı +æĶ¾ æĺł +è¨Ī çķ« +çݰ代 çī©æµģ +é¤IJ 廳 +æķħ å±ħ +大 大å°ı +大大å°ı å°ı +çī¹åĪ« 声æĺİ +éģį åıĬ +å¿ĥçIJĨ åĴ¨è¯¢ +è³ ´ +çĮ® è¡Ģ +å·²ç»ı è¾¾åĪ° +æīĵ æĭĽåij¼ +åıĮ è¾¹ +ä¸Ģæĸ¹éĿ¢ æĺ¯ +å´ĩ å°ļ +éĺ¿ å¯Į +éĺ¿å¯Į æ±Ĺ +æĮģ æľī人 +è± ģ +é£İ çŃĿ +åĬ¨ èį¡ +äºĨä¸Ģ ä¼ļ +äºĨä¸Ģä¼ļ åĦ¿ +ä¸ĩ 象 +çľĭ çĶµè§Ĩ +åįģä¸ī æĿ¡ +çĮĽ çĥĪ +è¦ģ ä¸įçĦ¶ +太æŀģ æĭ³ +å¼ķ çĪĨ +ç»ıè¿ĩ å¤ļå¹´ +游æĪı éĩĮçļĦ +é¾Ļ æ³ī +æłĩ éħį +è®ĵ ä»ĸåĢij +éĢł æŀĹ +åĮºåŁŁ æĢ§ +亿 ä¸ĩ +æĪĺçķ¥ å¸ĥå±Ģ +éķĩ æĶ¿åºľ +åĶ® 票 +çĶŁäº§ å·¥èīº +éķĩ åħļå§Ķ +ä¸Ńå°ı åŀĭ +æľ¨ èĢ³ +æ²³ è¾¹ +èĦ¾ èĥĥ +欢è¿İ æĤ¨ +åıĺ å¼Ĥ +缤 纷 +åŀĥåľ¾ 桶 +辩 è¯ģ +车 åºĵ +æ¯Ķ çİĩ +åħ´ æĹº +详ç»Ĩ äºĨ解 +å®ī å±ħ +çħ§ æĸĻ +æĸ¹ æīį +èµ ¦ +åĨ ķ +å¥Ķ èµ´ +å®Ŀ 鸡 +åľº åĿĩ +缮åīį æŃ£åľ¨ +åIJŀ åĻ¬ +è¿° èģĮ +æĩ µ +å¥ĩ çijŀ +ä»į å°Ĩ +èĪī 辦 +å·¥åķĨ å±Ģ +å¡ij èĥ¶ +åĬŀ å®ŀäºĭ +æĸ¹ æĸ¹éĿ¢ +æĸ¹æĸ¹éĿ¢ éĿ¢ +æĸĩåĮĸ èĬĤ +åħ¥ èģĮ +é¸ ¥ +ç©¿ éĢı +以 ä¹łè¿ijå¹³ +åį± éļª +æľ¦ èĥ§ +åİĨåı² æĢ§ +æķŀ å¼Ģ +ä¼Ļä¼´ åħ³ç³» +çŁ¿ åĮº +åĽ½éĻħ åľ¨çº¿ +ä¼łå¥ĩ éĩĮéĿ¢ +è¿ij äºĽ +è¿ijäºĽ å¹´ +åĬ£ åĬ¿ +æĶ»åĩ» åĬĽ +æĻº éĢł +ç¦ § +çİĭ åħĪçĶŁ +éĨ« çĶŁ +åĽĽ 项 +å®ŀ æĻ¯ +åĪĿ åĪĽ +å¿ĥ 裡 +æĻ¶ ä½ĵ +交 éĻħ +让 æ¶Īè´¹èĢħ +课 æĸĩ +æİĴ æ°Ķ +并ä¸į æĦıåij³ +缸 声 +第ä¸Ģ å±Ĭ +åİŁ èijĹ +éĽ ľ +没æľī 太大 +è¡¥ æ°´ +çī©æµģ ä¼ģä¸ļ +第äºĮ æī¹ +åħ¶å®ĥ éĹ®é¢ĺ +æİĮ éŨ +责任 å¿ĥ +é¤IJ åħ· +ç¾Ĭ æ¯Ľ +没æľī å¿ħè¦ģ +ä¹IJ åĽ¢ +è¿Ľ åŁİ +ä¸ĢçĤ¹ åĦ¿ +身 å½¢ +çļ®èĤ¤ çĹħ +æĺ ± +å¢ŀ èĩ³ +èģ² æĺİ +æıIJ è´¨ +ä½ĵèĤ² åľº +çŃ¹ 建 +é¬ Ĩ +车 çīĮ +éļĶ éŁ³ +è´Łè´£ åIJĮå¿Ĺ +丰 ç¡ķ +ä½Ľ éĻĢ +äºī åIJµ +åº ¶ +æ·¡ æ°´ +å°ı çĶ·åŃ© +ç§ģ èĩª +åĮĸ è¿Ľç¨ĭ +æĪĺ士 æĿ¥è¯´ +æ²¹ èħ» +èĦ±è´« èĩ´å¯Į +æĹ¥å¸¸ å·¥ä½ľ +交 èŀį +åĨľ è´¸ +åĨľè´¸ å¸Ĥåľº +åĵĪ çĻ» +çĶµ è´¹ +èµ ĺ +åıĮ èħ¿ +æĵĶ å¿ĥ +æĿ¥ 形容 +使åij½ æĦŁ +éĤ£ä¹Ī ç®Ģåįķ +èĬĻ èĵī +åĢŁæ¬¾ 人 +ç§Ģ 丽 +è®ĵ ä»ĸ +严åİī æīĵåĩ» +è³ ŀ +æļ « +çħ¤ æ°Ķ +çĪ¬ ä¸Ĭ +æ½ĩ æ´Ĵ +太 ä¹ħ +åij½ åIJį为 +è·¯ çĶ± +è·¯çĶ± åĻ¨ +é© ¯ +æıIJ æĹ© +æĬĹåĩ» çĸ«æĥħ +åĩ Ľ +交 åıĭ +éĶĢåĶ® æ¸łéģĵ +毫ä¸į çĬ¹è±« +èIJ¥ åľ° +çłĶ究 表æĺİ +é±¼ ç±» +æį¢ å±Ĭ +æİ¡ åıĸ +çī Ĩ +缼 å¼Ģ +沧 æ¡ij +åºŃ 审 +ç»ı æŁ¥ +åĬł å¼· +缸æ¯Ķ äºİ +ä¸ĵ çıŃ +ä½ĵ åŀĭ +被 害 +被害 人 +æĶ¶ 款 +åħ·æľī èī¯å¥½ +é«ĺå³° æľŁ +åģı ä½İ +åĦ Ł +åĨľä¸ļ ç§ijæĬĢ +çī¹æ®Ĭ æĥħåĨµ +å¦Ĥæŀľ çݩ家 +éķ¿ 约 +第åħŃ å±Ĭ +åħ¬å¼Ģ æĭĽèģĺ +åĪĩ æĸŃ +è¿« 使 +çĸĹ ç¨ĭ +第äºĮ ç§į +ä¸į åħį +å¹² èѦ +çŁ³ 榴 +åĹ £ +两 ç±» +çĪµ 士 +åŁİ乡 å±ħæ°ij +æѤ 项 +缴 è¾ĸ +缴è¾ĸ å¸Ĥ +åij¼ åºĶ +éĴ ¯ +ç¦ı å¾· +æľº 身 +æĵį åľº +æ¿Ĵ 临 +人群 ä¸Ń +èĤ¡ æ°ij +åŃ ½ +æ³ķ åħ° +é¨ İ +糯 ç±³ +æĢ» çļĦ +æĢ»çļĦ æĿ¥è¯´ +åħ¸ éĽħ +æĸ° éĻĪ +æĸ°éĻĪ 代谢 +缮 çĿ¹ +é¢Ħ è¨Ģ +è·Į çł´ +æĸ° ç¯ĩç«ł +æ¯Ĵ æĢ§ +åĸĿ èĮ¶ +æŁ¥ èİ· +亮 丽 +çĶŁäº§ åķĨ +æĶ¹ æĪIJ +为äºĨ æĽ´å¥½ +æ·± 交 +深交 æīĢ +æİ ĥ +ä¹Ļ èĤĿ +泸 å·ŀ +åħĪè¿Ľ æĬĢæľ¯ +è¾ĵ ç»Ļ +æķ£ æĪ· +æĢĿç»´ æĸ¹å¼ı +åºĹ 主 +è°ĭ æ±Ĥ +游æĪı æĬĢå·§ +ä¸Ģå¹´ 级 +çľ¼ è§Ĵ +ä¸Ńä»ĭ æľºæŀĦ +å·§ åIJĪ +éĺ² çĽĹ +导 è´Ń +æĪ Ĭ +æĽ´ éĢĤåIJĪ +åŁºæľ¬ ä¿¡æģ¯ +马 ä¸ģ +åħ»æ®ĸ åľº +åıį è¿ĩæĿ¥ +æݨ å´ĩ +å¯ĨåĪĩ åħ³æ³¨ +åŁºéĩij ç»ıçIJĨ +æĮī éĶ® +åĨħéĥ¨ æݧåĪ¶ +æĪIJåijĺ åįķä½į +æľ¯ è¯Ń +åĪ¶ æľį +åĪļ éľĢ +æ£Ģ ç´¢ +大大 æıIJé«ĺ +åģ¥åº· 管çIJĨ +èĩª æѤ +客æĪ· éľĢæ±Ĥ +丰 èĥ¸ +èµ· éĩį +èµ·éĩį æľº +æ¬ł 缺 +æ¡Ī åŃIJ +æĥħ人 èĬĤ +åħļ æł¡ +è¢ ľ +该 åī§ +迷失 ä¼łå¥ĩ +ç»ļ 丽 +åķ ª +æĹł ç§ģ +éĢ² ä¸ĢæŃ¥ +第ä¸Ģ ç«ł +åĻ¨ åħ· +åĨľ èµĦ +確 實 +åºı åĪĹ +娱ä¹IJ å¹³åı° +èŀįèµĦ 秣èµģ +èµĦæºIJ åħ±äº« +èģ½ åĪ° +æIJŀ å¾Ĺ +继ç»Ń ä¿ĿæĮģ +åIJ¯ èĴĻ +çľ º +ä¸Ŀ è·¯ +设æĸ½ 建设 +æİ¥ åľ° +æİ¥åľ° æ°Ķ +第ä¸ī åŃ£åº¦ +åŁº è°ĥ +åıij éŁ³ +社ä¼ļ èµĦæľ¬ +éĽĩ 主 +è¿ŀ èĥľ +没 åķ¥ +å» ¢ +赶 èµ´ +æ¼Ķ åĮĸ +åı¤ æĢª +çİĭ çĪ· +é¢Ħ åħĪ +å¼Ģ åħ· +åĽŀ é¦ĸ +åľ°ä¸ĭ æ°´ +å°ıç¼ĸ ä¸Ģèµ· +èµİ åĽŀ +åľ° è²Į +åĪĿ ä¸ī +åı¯ çĶ¨äºİ +éģĹ è¿¹ +è¿Ļ æī¹ +èĸª æ°´ +å¿ħçĦ¶ ä¼ļ +æ² ½ +éį ĭ +第ä¸Ģ éĥ¨ +åĪĬ çī© +å®ŀ ä¾ĭ +æ¸ħ åĩĢ +ä¸Ĭ èµĽåŃ£ +åĽ¾ 表 +éĤ® è½® +åĵª 裡 +缸 è§ģ +æī° ä¹± +æ¯ı æ¯ı +è¿Ļ è¾ĪåŃIJ +ç¡« éħ¸ +äºī 缸 +溯 æºIJ +åĩº ä¼Ĺ +çİī çŁ³ +åħ± çĶŁ +æŶéĹ´ 段 +éĩįè¦ģ æĮĩ示 +æ¶Īè´¹ éľĢæ±Ĥ +éķ¿ éķ¿ +éķ¿éķ¿ çļĦ +å®ī æĬļ +å¢ŀ é«ĺ +æľ¬ è½® +亲 çľ¼ +é£İ æ³¢ +èĢģ å¦Ī +æĶ¶è´¹ æłĩåĩĨ +åĨħ éĻĨ +æĮ¥ åıij +åįĩ åѦ +èĥ¸ åīį +åģı è¿ľ +纯 æ´ģ +æĸ½å·¥ åįķä½į +身 ä»· +è´¢ åĬĽ +çº ¶ +è£ħ çĶ² +æĺ¾ç¤º åĻ¨ +毫 åįĩ +æ·± çŁ¥ +èĢ¶ ç© +èĢ¶ç© Į +è¾ĥ éĩı +åľ¨ è¿ĩ渡 +åľ¨è¿ĩ渡 æľŁ +èĮ Ĺ +ä¸Ģ个 æĺŁæľŁ +èĬ · +è´¿ èµĤ +æ¿ ķ +æĩĤ äºĭ +ç§ § +åħħ å½ĵ +åĽ½ ç«ĭ +èĬ± çĵ£ +éĤĦ è¦ģ +åħ¬ åľĴ +触 åĬ¨ +æ³° å·ŀ +ä»Ģä¹Ī æł· +æ»ĭ åħ» +è¯Ħ åĪ¤ +æĮ¥ æīĭ +èĦ Ī +姥 姥 +è¿IJ è´¹ +æ¯ħ åĬĽ +å¿ĥ æĻº +ä¸į æİĴéĻ¤ +第ä¸ī 代 +éĢĢ è´§ +æĺŁ éĻħ +æ°¸ åĪ© +æĬ¤ åį« +çıŃ è½¦ +è¨Ģ è¡Į +ç¹ ª +主åĬ¨ æĢ§ +å·¥ç¨ĭ è´¨éĩı +éĥĬ åĮº +ä¸Ģ æłĭ +ä½Ĩ å®ŀéĻħä¸Ĭ +ä¸ī大 èģĮä¸ļ +åij¼ åı« +女 åħĴ +è¯ģåĪ¸ æĬķèµĦ +èĢĥ æħ® +çĤ« èĢĢ +æ²» 好 +åĺ ¶ +èĥ ¤ +åħīä¼ı åıijçĶµ +åĩł æŃ¥ +æīĢ æīĢ +æīĢæīĢ éķ¿ +çħ§ æł· +åĵ¥ 们 +è¯ Ľ +è¿Ļä¸Ģ åĪ» +çŁ¿ çī©è´¨ +ä¸įå¾Ĺ å·² +åIJĮ 缣 +ç»Ĩ å¾® +è·¯ èĻİ +çĻ¾ èĬ± +æ·· æ²Į +ä¸Ĭæµ· è¯ģåĪ¸ +éĢĢ ç¨İ +èµŀ åı¹ +æī®æ¼Ķ 游æĪı +åIJį åĪĹ +åIJįåĪĹ åīį +åIJįåĪĹåīį èĮħ +ç±³ å°Ķ +ä»Ģä¹Ī åİŁåĽł +å®īåħ¨ ä¿Ŀéļľ +ä¸Ģåıª æīĭ +ä¹³ ä¸ļ +ä¸į çĶĺ +æĥħ åķĨ +æĮ¡ ä½ı +åİŁåĽł ä¹ĭä¸Ģ +è¿Ļ 两天 +çĥĺ çĦĻ +è± ¬ +ä½ł 以为 +没 è§ģè¿ĩ +åĵªå®¶ 好 +åīį ä»» +è¿Ľ è´§ +éĢĢ åĽŀ +串 èģĶ +èĩ³ æĸ¼ +åĨ° æ·ĩ +åĨ°æ·ĩ æ·ĭ +æŁ¥çľĭ 详æĥħ +çı¾ 實 +æݨ æµĭ +æİ¥ æīĭ +éļ¶ å±ŀäºİ +åŁİå¸Ĥ 群 +æĿİ åħĪçĶŁ +çŁ¿ æ³īæ°´ +çī¹ ä»· +æĽ´å¤ļ 精彩 +ç¨ĭ å¼ı +读 æĩĤ +å±ı èĶ½ +奥 æŀĹ +奥æŀĹ åĮ¹ +奥æŀĹåĮ¹ åħĭ +红 èĸ¯ +å¥ ® +å®Ŀ çİī +網 絡 +è² § +欧 å¼ı +çĻ½ ç³ĸ +èĩªçĦ¶ çģ¾å®³ +åijĬè¯ī 她 +å» ļ +çĤ¹åĩ» æŁ¥çľĭ +é£İ 湿 +èµĦ产 éĩįç»Ħ +ä¹Łä¸į ä¾ĭå¤ĸ +åįĬ 个å°ıæŶ +åIJ¸å¼ķ æĽ´å¤ļ +æŶéĹ´ èĬĤçĤ¹ +æĶ¶ 纳 +åIJ¸ æ¯Ĵ +èĢģ 乡 +çIJ ħ +æľĢ çµĤ +åıį æĦŁ +çĶ¨ 微信 +çĶ¨å¾®ä¿¡ æī« +éĢŁ çİĩ +大 çĨĬçĮ« +åı¯ æĥ³ +åı¯æĥ³ èĢĮ +åı¯æĥ³èĢĮ çŁ¥ +åĴ § +èµ° åħ¥ +碳 éħ¸ +èĮĥ åĨ° +èĮĥåĨ° åĨ° +被 åĪ¤ +积æŀģ æݨåĬ¨ +足 足 +ç²Ĵ åŃIJ +大 å®Ĺ +大å®Ĺ åķĨåĵģ +ç½ij绾 ç§ijæĬĢ +æĽ¼ åŁİ +å·² ä¹ħ +å·²ä¹ħ çļĦ +秦 çļĩ +秦çļĩ å²Ľ +ä»» æķĻ +åĶ¯ ç¾İ +æ·¡ åĮĸ +æ¡Ĥ èĬ± +çŁ¥è¯Ĩ åĪĨåŃIJ +æĩĴ å¾Ĺ +主 åħ¬ +设计 çIJĨ念 +è³ º +æīĢ æıIJä¾Ľ +æīĢæıIJä¾Ľ ä¹ĭ +æĶ» åħĭ +åĤ ¾ +è¯Ń æ³ķ +åįĥ åı¤ +éĸĭ æĶ¾ +第ä¸Ģ èĬĤ +éĤĦ æ²Ĵ +éĢĥ çĶŁ +æ³ Ĺ +åİ¿ å§Ķ书记 +ä½ľèĢħ æīĢæľī +çħ ½ +ç» ħ +æł ħ +æľ´ ç´ł +çijķ çĸµ +åĮħ åĮħ +æ°ij主 åħļ +ä¸į è¿ľå¤Ħ +å¥ĩ å¼Ĥ +åĺ» åĺ» +æī ¼ +ç¿» å¼Ģ +æĢİ èĥ½ +éģ´ éĢī +解 éĩĭ +å¹¼ ç¨ļ +è¦ģ 好好 +趴 åľ¨ +ç´¢ åıĸ +ç»Ī çĶŁ +åħ¨ æµģç¨ĭ +éģ© çķ¶ +åįıè°ĥ åıijå±ķ +æĬ¥ ä»ĩ +ç§ijæĬĢ åĽŃ +ä»Ģä¹Ī éĥ½ä¸į +æľĢåIJİ ä¸Ģ次 +ç»Ļ人 ä¸Ģç§į +æł¸ å®ļ +被 åĪĹåħ¥ +æĦı æĥ³ä¸įåĪ° +èĢĥ æŁ¥ +åľ¨æѤ ä¹ĭåīį +æīĵ çIJĥ +è¶ĬæĿ¥è¶Ĭ å°ij +å®ļ å¾ĭ +è¡ĮæĶ¿ æľºåħ³ +ä½ıæĪ¿ åħ¬ç§¯ +å°ıå§IJ å§IJ +ä¸ī èı± +ä¿® è¡¥ +èŀĥ èŁ¹ +西 çĶ² +æĢ ł +çŃī å¤ļ项 +产ä¸ļ éĽĨèģļ +ä»·æł¼ ä¸Ĭ涨 +åħ¬åħ± åľºæīĢ +è¢ĭ åŃIJ +æĨ§ æĨ¬ +çļĦæĸ¹å¼ı æĿ¥ +åĪ° è´¦ +çģ ½ +å·´ èı² +å·´èı² çī¹ +æ¼Ķ ä¹ł +èŃ¦ç¤º æķĻèĤ² +çķı æĥ§ +å¼ķ æµģ +æĶ¶ æĶ¯ +å±Ĥ åĩº +å±Ĥåĩº ä¸į +å±Ĥåĩºä¸į ç©· +æijĩ æ»ļ +辦 çIJĨ +纵 è§Ĥ +æķij æµİ +家 éĥ½çŁ¥éģĵ +åĮ ¯ +å°ı 鸣 +ä»» åĭĻ +计 åħ¥ +ç«ŀ éĢī +å¼ĢèįĴ æĹ¶æľŁ +åij¨ æģ© +åij¨æģ© æĿ¥ +交 ç»ĩ +çķ¢ æ¥Ń +æł¹æį® èĩªå·± +æĸ°äºº çݩ家 +åѵåĮĸ åĻ¨ +éĩĩ æļĸ +å¹³åĿĩ æ°´å¹³ +åħ¬å¼Ģ 课 +失 åĪ© +伺 æľį +çĬ ģ +忽 æĤł +主è¦ģ éĽĨä¸Ń +æ¤į æłij +æ¯Ĺ éĤ» +èĩº çģ£ +åĩºåĽ½ çķĻåѦ +æĬĹ éľĩ +æĥ© æĪĴ +å¹´åºķ åīį +åĴ¸ éĺ³ +æ°ij å±ħ +大çIJĨ çŁ³ +éĿ ³ +éķ ĸ +æ¸ħ è¿ľ +è£ħ è½½ +èĩ Ģ +å½± ä¸ļ +å¼Ł åħĦ +æĤ² è§Ĥ +çĿĢçľ¼ äºİ +æįį åį« +åī¥ 夺 +ç¯ Ĩ +å¾Ī éķ¿æŶéĹ´ +è¥ Ł +第ä¸Ģ çĻ¾ +ä¸ĢåĪĨ éĴ± +æĸ°éĹ» è®°èĢħ +éķ· æľŁ +æ³ķ æĪĺç»ĦåIJĪ +è°ģ çŁ¥éģĵ +èħ° éĥ¨ +æ±ī åł¡ +åħ¥ çĿ¡ +åįĸ æİī +æ¶Īè²» èĢħ +æĥ¯ ä¾ĭ +æĥ³ äºĨ +æĥ³äºĨ æĥ³ +èĢģæŧ å°ıåĮº +ä¼ł è¨Ģ +åĪĨæķ° 线 +æµģ 泪 +ç»Ħç»ĩ é¢Ĩ导 +äºļ åĨĽ +å¢ŀåĢ¼ æľįåĬ¡ +å¾ ¹ +ä¼ ¶ +äºĽ 许 +å¸ĥ èݱ +强 æĤį +宫 å»· +绿 èĮ¶ +åĮ ¡ +å¾Ī æŃ£å¸¸ +æĺ¥ å¤ı +æ¯ Ļ +è¯Ħ æ¯Ķ +åĩ¡ äºĭ +æĬī æĭ© +åĢĴ éľī +éĩį 度 +åįıä¼ļ ä¼ļéķ¿ +忧 èĻij +ä¸ĭ ä¸Ģç¯ĩ +沪 æ·± +æĪ İ +æīĵ ä»Ĺ +åįĪ é¥Ń +å¹´é¾Ħ 段 +ä¸ŃåĽ½ 足çIJĥ +设计 æĸ¹æ¡Ī +åºĶçĶ¨ æŁ¥çľĭ +é¢Ħ æĸĻ +åĹ ¡ +ç¥ĸ çĪ¶ +çļĦä¸Ģ åijĺ +æ´Ĺ å¹²åĩĢ +åİĨåı² æĸ° +åİĨåı²æĸ° é«ĺ +çĭ¬ åħ· +æħĭ 度 +æīĵ 交 +æīĵ交 éģĵ +é»Ħ çŁ³ +çĽ¼ æľĽ +çī§ åľº +转 弯 +åįĩ åįİ +åĨį ä¹Łæ²¡æľī +èĭ± æīį +æĽ´ åIJį为 +åĢŁ çĶ¨ +çºł éĶĻ +ç»Ŀ对 ä¸įä¼ļ +çİĭ çīĮ +çĽĨ åľ° +失 è°ĥ +好 象 +é³ ¥ +ä¿Ŀ ä¿® +åĽĽä¸ª èĩªä¿¡ +头 çļ® +åİŁ åīĩ +æĬ¥ æ¡Ī +奴 éļ¶ +å³ Ļ +è°ĥ æĸĻ +ä¹Ł 許 +èIJ½ åĪ° +èIJ½åĪ° å®ŀ +èIJ½åĪ°å®ŀ å¤Ħ +çĦļ çĥ§ +çĶŁæ´» çݯå¢ĥ +åºĶ åıĬæŶ +è¶Ĭ è¿ĩ +æĦŁ è¬Ŀ +æĻ¯ å¾· +æĻ¯å¾· éķĩ +çĬ Ģ +身 éĤĬ +ç¨İåĬ¡ æĢ»å±Ģ +åĩĢ åľŁ +ä¾µ åįł +åĬ¨ å·¥ +å¹´ ä¹ĭ +å¹´ä¹ĭ ä¹ħ +第äºĮ èĬĤ +åĬ¨çī© åĽŃ +第ä¸Ģ 书记 +éħ ļ +çĶŁäº§ 设å¤ĩ +æŁIJç§į ç¨ĭ度 +åľ Ń +åĩŃåĢŁ çĿĢ +éĺħ è§Ī +çĻ½ æ²Ļ +æ²¹ çĥŁ +çªģçł´ åı£ +åıĹ å½±åĵį +åı¯ä»¥ æĽ´å¥½ +å³° åĢ¼ +æĿĤ è´¨ +宿 è¿ģ +çĽĺ æ´» +æ¿Ģ èµ· +åĦ¿ ç§ij +åĿIJ èIJ½åľ¨ +æĮª å¨ģ +æµ· å²Ľ +绣 绣 +éĻ ¨ +ä¼ĺ äºİ +å°Ī 家 +ä¸Ģ éĤĬ +èIJ Ĭ +äºĨä¸Ģ åı£ +æ²ĥå°Ķ æ²ĥ +æŃ£å¸¸ 使çĶ¨ +æĻ®éģį åŃĺåľ¨ +丰 满 +çĶ» åį· +åºĶ æĶ¶ +åºĶæĶ¶ è´¦ +åºĶæĶ¶è´¦ 款 +å®Įæķ´ çĥŃ +å®Įæķ´çĥŃ æ¦ľ +注 è§Ĩ +çĨ Ħ +èº ¬ +éĶĢåĶ® 人åijĺ +è¶ĭ åIJij +çĦ¦ æĢ¥ +åįģå¹´ åīį +ä¼łç»Ł 产ä¸ļ +質 éĩı +åĩ¤åĩ° ç½ij +èµĦæºIJ æķ´åIJĪ +æ¶Į åħ¥ +æĸĩåĮĸ ä¼łæĴŃ +çķĮ 第ä¸Ģ +æ°´ æ³µ +宫 殿 +æİ¢ 寻 +ä¿® åīª +æĦı è¦ĭ +ç´Ĭ ä¹± +æĽ ī +çĻ½ è¡£ +èĻİ åį« +ç´§ æī£ +å¤Ħå¤Ħ éķ¿ +åĪĽå»º å·¥ä½ľ +红 æŀ£ +饼 å¹² +äºĨ åįĬ天 +ä¼ļå½±åĵį åĪ° +çĽ¸ä¿¡ 大家 +èħ¾ é£ŀ +å°± å¦ĤåIJĮ +ä¸ĭéĿ¢ å°ıç¼ĸ +æ°ijèIJ¥ ç»ıæµİ +æĻ ¦ +è£ħ æī® +é»ij å¤ľ +常 å¾· +å·¥ä¸ļ 大åѦ +æĺİ çŁ¥ +éĺŁåijĺ 们 +åIJ¬ 课 +æ¯ı éļĶ +羣æĺ¯ å¤ª +åIJĪä½ľ åħ±èµ¢ +çIJĨ åıij +æīį å¹² +çľĭ èµ·ä¾Ĩ +殿 ä¸ĭ +å®ī éĺ³ +æīĢ 产çĶŁçļĦ +éĽĩ ä½£ +æĬ¬èµ· 头 +æį® æĬ¥éģĵ +éļĨéĩį 举è¡Į +交 éĶĻ +è¶ħ é¢Ŀ +åĮĸ çĸĹ +é¡ Ĩ +纵 æ·± +çĪ±åĽ½ 主ä¹ī +éĻ¢ åī¯éĻ¢éķ¿ +è® ³ +羣æŃ£ åģļåĪ° +åѤ åįķ +èĩªçĦ¶ èĢĮ +èĩªçĦ¶èĢĮ çĦ¶ +ä¿® 身 +èĬ ¹ +æģ¯ æģ¯ +æģ¯æģ¯ 缸åħ³ +驾 æł¡ +æİ© 饰 +æ³½ è¿ŀ +æ³½è¿ŀ æĸ¯åŁº +举 æŃ¢ +管çIJĨ ä½ĵåĪ¶ +åħ¶ä¸Ń ä¹ĭä¸Ģ +æĿ¾ å¼Ľ +æĭ¦ æĪª +åį« åģ¥ +åį«åģ¥ å§Ķ +ä»İ åݻ年 +åĤ ¢ +è´Ń 票 +åĽ¾ æłĩ +æ²³ 西 +æ°ijæĶ¿ å±Ģ +ç§ģ èIJ¥ +å¤ĸåĽ½ è¯Ń +å¹² è´§ +æĵ¦ æĭŃ +åľ° ä¸Ń +åľ°ä¸Ń æµ· +æµĵ æµĵ +æµĵæµĵ çļĦ +å§ĭ 建 +å§ĭ建 äºİ +ç¶ĵ æŃ· +è·¯ æ¼Ķ +æļ´ é£İ +åŁº è¾ħ +æī¶è´« å·¥ä½ľ +ä¸Ģ缴 å¤Ħäºİ +æĥħ 趣 +äºĮ åŃ£åº¦ +åİĮ æģ¶ +顺åĪ© å®ĮæĪIJ +æŁ¥ å°ģ +顶 端 +ä¸į åŃķ +ä¸Ģ大 åłĨ +被 æ·ĺæ±° +æĺ¯ çĶ¨æĿ¥ +æľĢ åIJĪéĢĤ +亮 çľ¼ +并ä¸įæĺ¯ å¾Ī +ç§ijçłĶ éĻ¢ +ç§ijçłĶéĻ¢ æīĢ +ç² Ł +é¢Ī éĥ¨ +é»ĺé»ĺ åľ° +é«ĺä¸Ń çĶŁ +æĹıèĩªæ²» åİ¿ +æķĻåѦ è´¨éĩı +æĪĺ çģ« +åĿİ åĿ· +æIJŃ ä¹ĺ +è¯Ĺ æĦı +åĪij èѦ +åĩº æ±Ĺ +åįģåħŃ æĿ¡ +请 åıĬæŶ +åĨľä¸ļ 大åѦ +èIJ½ åı¶ +æĢ» èĢĮè¨Ģ +æĢ»èĢĮè¨Ģ ä¹ĭ +æĿľ åħ° +æĿľåħ° çī¹ +éĻª ä½ł +åħ¬ æĬ¥ +çķĻè¨Ģ æĿ¿ +éĺħ åİĨ +競 çĪŃ +ç»Ļ åĪ«äºº +æĹ¥æĬ¥ 社 +åĿIJ èIJ½ +åĿIJèIJ½ äºİ +éĩij åŃĹ +éĩijåŃĹ å¡Ķ +åĽ ¤ +è¯Ŀ åī§ +æĮģç»Ń æİ¨è¿Ľ +æ¼ı æ°´ +詳 ç´° +æĢĢ æĬ± +åıĺ å¹» +饥 饿 +éļIJ 身 +个 èµĽåŃ£ +åĵ¡ å·¥ +æģ¢å¤į æŃ£å¸¸ +äºĨ 好å¤ļ +æĺŁ å·´ +æĺŁå·´ åħĭ +åħī çݯ +å¸ħ åĵ¥ +çĻ½ éĽª +ç¨į ç¨į +计 æıIJ +æĦĽ æĥħ +éİ ĸ +ä¿¡ éĺ³ +è§Ģ å¯Ł +å¦Ĥæŀľä½ł æĥ³ +缸æ¯Ķ ä¹ĭä¸ĭ +解 å¼Ģ +æīĵåį° æľº +身 躯 +ç²¾ç¥ŀ æĸĩæĺİ +èĤ¡ æĮĩ +å¾® åĪĽ +红 èĮ¶ +èĩ´ çĻĮ +æģ© æĸ½ +èħ¿ éĥ¨ +大åŀĭ å¤ļ人 +å®ī åĢį +è¾ħ导 åijĺ +èĪª éģĵ +å¸ĥ å°Ķ +åįĹå®ģ å¸Ĥ +ä¸ĬçıŃ æĹı +侧 ç»ĵæŀĦæĢ§ +追 éļı +å½ĵåľ° æĶ¿åºľ +èµ° åĩºæĿ¥ +éĩijèŀį ä¸ļ +丼 书 +é¡¹çĽ® ç»ıçIJĨ +è¿ĩ æĪ· +骨 æŀ¶ +è¡ Ļ +ä»Ģ 麽 +èħ ĭ +è¦ģ 害 +åľ¨ åºĬä¸Ĭ +代è¨Ģ 人 +並 å°ĩ +åIJĦ个 æĸ¹éĿ¢ +è°´ è´£ +åħ± æĮ¯ +åį³å°Ĩ åĪ°æĿ¥ +èĤº çĻĮ +ä¾Ľ éĶĢ +丼 æŀĹ +èµ ĥ +åįģä½Ļ å¹´ +åĭĺ æİ¢ +飵 åij³ +èĭ¦ ç¬ij +æľĢ大 ç¨ĭ度 +éĩįçĤ¹ åħ³æ³¨ +ä¹ĭ 举 +满 æĢĢ +åıĹåĪ° å½±åĵį +æĭĽ æĬķæłĩ +è¡¥ é½IJ +西 红 +西红 æŁ¿ +é¬ § +è£ħ åį¸ +éĤ» éĩĮ +èĤĩ äºĭ +æİĴ æ¯Ĵ +åѤ åĦ¿ +鼶 è·Ŀ离 +å®ŀ å¹² +çľĭ æŁ¥çľĭ +æĶ¶è´¹ ç«Ļ +ç» · +åħ¬çĽĬ æĢ§ +éĢĴ ç»Ļ +æĶ» æīĵ +æĺŁçº§ éħĴåºĹ +æĺİ åªļ +çį¨ ç«ĭ +è¯Ŀè¯Ń æĿĥ +ä¸ĢæŃ¥ ä¸ĢæŃ¥ +书æ³ķ 家 +æľªç»ı æİĪæĿĥ +çŁ³ èĨı +åĩŃ ä»Ģä¹Ī +çļĦ æĹ¥ +çļĦæĹ¥ åŃIJéĩĮ +诱 人 +çĻ¾åĪĨ çĻ¾ +èĪĪ 趣 +å¼ł åħĪçĶŁ +èĢģçĪ· åŃIJ +æ³¢ çī¹ +åŁºéĩij 份é¢Ŀ +æ²Ļåıij ä¸Ĭ +å¥ĭæĸŠ缮æłĩ +æ°¢ èĥ½ +æ²ĥå°Ķ çİĽ +義 åĭĻ +éŁ³ ç®± +æ²ī 浸 +æ²ī浸 åľ¨ +èĭ± åľĭ +çģ¯ çģ« +è¿Ľ 项 +两 端 +ä¹Ķ 丹 +èĦ¸ é¢Ĭ +åıijå±ķ æ½ľåĬĽ +åĭķ ä½ľ +åĵĪ ä½Ľ +å®´ ä¼ļ +æ§ į +ç«ĭ å¿Ĺ +ç¡ķ士 åѦä½į +åĭĭ ç«ł +è¿Ļ åľºæ¯ĶèµĽ +æĮģ å¹³ +éķĢ éĶĮ +èĭ± çī¹ +èĭ±çī¹ å°Ķ +æķĻ èģĮå·¥ +åĬŁ åĬĽ +该 æ¡Ī +ä¸Ģ æ¢Ŀ +åĺī å¹´ +åĺīå¹´ åįİ +è¿« ä¸įåıĬ +è¿«ä¸įåıĬ å¾ħ +è¿Ļ个 æĹ¶ä»£ +精彩 æĴŃæĬ¥ +人 èĦ¸ +人èĦ¸ è¯ĨåĪ« +æ£Ģå¯Ł å®ĺ +å°ı èħ¿ +éĨĴ 缮 +åħļ æĢ» +åħļæĢ» æĶ¯ +æĪ Ł +èĮ« çĦ¶ +è±Ĩ æµĨ +主 æ²» +éĿĴæµ· çľģ +åĪijäºĭ 责任 +çł ° +ä¹ĭ æ¬ĬåĪ© +äºĶ å®ĺ +è¿· æĥij +åħ¥ åºĵ +家 纺 +å¼¹ ç°§ +åįģäºĶ æĿ¡ +ç»Ļ å®Ŀå®Ŀ +èĪªç©º èĪªå¤© +å¾Ģ å¤ĸ +å¼ķ åĬĽ +çľ¼ çļ® +æ¶ī 足 +æĿ¥ 宾 +åľ¨çº¿ è§Ĵèī² +çĥŃ éĶĢ +æµģ éĢĿ +泡 泡 +éĻį å¹ħ +è´ŁéĿ¢ å½±åĵį +红 楼 +红楼 梦 +éļĶ çĿĢ +ä¾¥ 幸 +许 ä¹ħ +åĴĮ çĿ¦ +èŃ ½ +使çĶ¨èĢħ æĪĸ +ä¹° åįķ +è¿ ´ +é£İ æīĩ +æķĻ 師 +æ¡ĮåŃIJ ä¸Ĭ +å¾Ī æ¼Ĥ亮 +åł± å°İ +第ä¸Ģ åŃ£åº¦ +ç©© å®ļ +æĤ² åĵĢ +çĿĢåĬĽ æīĵéĢł +æĮ Ł +è·¯ æ¡¥ +åij IJ +åľ£è¯ŀ èĬĤ +çļĩ åŃIJ +ä»ĩ æģ¨ +éħĿ éħ¿ +ä¸į éĹ´ +ä¸įéĹ´ æĸŃ +æĮĩ å°ĸ +ä¸ŃåĽ½ ç½ij游 +åŀ £ +æĦıè§ģ 建议 +æ¯ħ çĦ¶ +亮 度 +èģĶ è°Ĭ +å½ķ åħ¥ +åĦ ² +å¨ĺ 家 +ç§ij å°Ķ +ä¹Łæ²¡ ä»Ģä¹Ī +æł¹æį® ä¸įåIJĮ +åı¶ ä¿® +åĢ¼ å®Ī +æľ« 端 +åĪ ¨ +åĤµ åĭĻ +èģ¯ åIJĪ +å¥ĩ å¹» +èĻļ æŀĦ +é»Ħ æĺı +å¹³ åĿ¦ +æµģ æ°ĵ +æĸ° åŁºå»º +æĮ½ æķij +åįİ å°Ķ +åįİå°Ķ è¡Ĺ +æľĢ åıĹ欢è¿İ +ç»Ń 约 +å¼Ĭ 端 +éŃĶ æ³ķå¸Ī +éŃĶæ³ķå¸Ī åĴĮ +åħ·ä½ĵ åĨħ容 +çIJī çĴĥ +æī© 容 +èĮ¶ åĽŃ +主ä¹ī èĢħ +ç«ĭ éĿ¢ +æİ¥åıĹ éĩĩ访 +åĩº åħ¥å¢ĥ +ç§ij åįı +éĴ ³ +çµIJ æ§ĭ +ç»ĵæŀľ æĺ¾ç¤º +åı° è´¦ +å°± æĿ¥çľĭçľĭ +èĩª æķij +åıį æĩī +åİ» åĵªåĦ¿ +è¿Ļ é¦ĸ +è¿Ļé¦ĸ æŃĮ +åIJ¬ ä¼Ĺ +å¤ĸ 壳 +ä½ĵèĤ² é¦Ĩ +實 æĸ½ +èŀº ä¸Ŀ +æĭī åįĩ +çĮĽ åľ° +åħ¨åĽ½ 人æ°ij +æĤī å°¼ +æĹı 群 +åĽ¢ åijĺ +两个 å°ıæŶ +åľ¨ çݩ家 +åľ¨çݩ家 ä¸Ń +çĶľ çĶľ +æĬķ è¡Į +åįĶ æľĥ +éĻ ¡ +åĬłå·¥ åİĤ +æ¦Ĩ æŀĹ +æŃ» è§Ĵ +åĨħ å¹ķ +æīĢæľī æĥħèĬĤ +åĪ· åį¡ +æ°´ èĤ¿ +èĥĥ åı£ +å«Į å¼ĥ +æ²® 丧 +ä¸īå¹´ 级 +æ¶Ĥ å±Ĥ +å¿ĥ 仪 +å¿ĥ仪 çļĦ +å¤ Ń +é¦ĸ è½® +æĹłè®ºæĺ¯ åħ¶ +éĢı æ°Ķ +äºĮ åįģäºĶ +ç® « +åĬŁ åĬ³ +çŃ¾ ä¸ĭ +æ²ī è¿· +æķij åij½ +éĹª éĹª +åIJĥ äºı +å±ķ åĵģ +åį³æŶ åıijçĶŁ +ç¶ ľ +綾 åIJĪ +æłĩ æĺİ +çľĭ çĶµå½± +åħ¬ ç«ł +éĺ¿ æ£® +éĺ¿æ£® 纳 +身 åĪĽéĢł +身åĪĽéĢł çļĦ +æ¸Ľ å°ij +åĢ¼å¾Ĺ åħ³æ³¨ +鼶åĶ® åķĨ +æįĨ ç»ij +è¸ı åħ¥ +èĽ Ł +æŁ´ 纳 +èĢģ åħµ +绿èī² çݯä¿Ŀ +é¹ Ń +麻 æľ¨ +æıŃ çīĮ +è¿Ļ款 车 +ç¾İ å¾· +ç¾İå¾· åħ¬åı¸ +æ¶ § +è°ģ çŁ¥ +æ´ĭ èij± +æ¯į æł¡ +ä¸Ģ éĹª +çĶ· 主è§Ĵ +æĹłçº¿ çĶµ +å±ł å®° +æĺ¯ éŁ©åĽ½ +æĺ¯éŁ©åĽ½ 娱 +容 è²Į +åĿĩ 使åħ¶ +太 å¿« +å¹´ çĶ± +å¹´çĶ± 缼 +èĭ¦ èĭ¦ +åĬĽ è¿ĺæĺ¯ +åĬĽè¿ĺæĺ¯ èĩª +æĨ © +èģ¯ 絡 +åĶ ¾ +åħ·æľī æĪĺ士 +追 éĹ® +åłĨ æĶ¾ +åıį 驳 +å®ŀäºĭ æ±Ĥ +å®ŀäºĭæ±Ĥ æĺ¯ +åѸ éĻ¢ +åįģ åĩłä¸ª +æķij æĬ¤ +æķijæĬ¤ 车 +ç½ij绾 ä¼łæĴŃ +åįģåħ« å±Ĭ +éĥ¨ åī¯ +éĥ¨åī¯ éĥ¨éķ¿ +çĹ´ è¿· +管çIJĨ æĿ¡ä¾ĭ +èŀį 为ä¸Ģä½ĵ +æĢ» 产åĢ¼ +è³ ĵ +ä¸ĥ æĺŁ +çıŃ ç»Ħ +绣 é¢Ĩ +请 大家 +éĩij éĻµ +èĪħ èĪħ +æµ· æ¹¾ +æĸ½ çŃĸ +享 èªī +éº ¥ +端 åįĪ +绿 åŁİ +確 ä¿Ŀ +å·´ æĭī +åĨĴ çĿĢ +æħ· æħ¨ +个人 è§ĤçĤ¹ +ä¹Ļ çĥ¯ +ç¡ħ è°· +éĸĭ å±ķ +å°ļ 书 +åĿļ 飧 +åº µ +èĢģ é¾Ħ +èĢģé¾Ħ åĮĸ +羨 çľ¼ +绿 æ°´ +绿水 éĿĴå±± +书 é¦Ļ +主åĬĽ åĨĽ +æīįæĺ¯ çľŁæŃ£ +æĬ¢ åħĪ +æĪIJå°± æĦŁ +éĩį æŀĦ +éĴ¢ åİĤ +æĪIJ 份 +èĬ± 纹 +ä¹ĭ äºī +å¹² ç»Ĩèĥŀ +æĹ¢ åı¯ä»¥ +ç¹ģ çIJIJ +æĦļ èł¢ +éĿŀ常 æĺİæĺ¾ +ä½ĵ 彩 +æĬĢ æ³ķ +æĿĨ èıĮ +å¹¿æ³Ľ åħ³æ³¨ +åĮĹ å®ĭ +å§Ĭ 妹 +åįı åĬŀ +æ·® åįĹ +çĥ ı +æ´Ĺ èĦ¸ +åıĹ è®¿ +åıĹ访 èĢħ +éĩįè¦ģ åĽłç´ł +å½±è§Ĩ åī§ +综èīº èĬĤ缮 +èľķ åıĺ +äºĮ 线 +äºĮ线 åŁİå¸Ĥ +ä¼Ĭ å§ĭ +çıĬ çijļ +èĩª æŁ¥ +åħ¥ åĽŃ +åĩ¶ æīĭ +åħ¬ è¯ī +éģĩ éļ¾ +éĩĩçŁ¿ çŃī +èĩª çIJĨ +åĸ· æ¶Ĥ +æī© åħħ +éĢı è§Ĩ +é«ĺéĢŁ å¢ŀéķ¿ +åĽ¾ çĶ» +ç¾ ¹ +èĤĩ åºĨ +è¾ľ è´Ł +èµĶ ä»ĺ +è· ¡ +åģ¥åº· æĪIJéķ¿ +以ä¸Ĭ åѦåİĨ +åıĸå¾Ĺ 以åıĬ +æ²ī 积 +åįģä¹Ŀ å±Ĭ +缸éĹľ æľįåĭĻ +æī§ åĭ¤ +åī¯ åİ¿éķ¿ +å¯ ° +åģľ æ»ŀ +æ·¹ 没 +çŁ³ çģ° +çį ¸ +åĢ ¦ +ç¾İ åªĴ +æķĻ æ¡Ī +åĬł çĽĸ +åħ¬å¼Ģ èµĽ +å¥ł åŁº +æĺĨ èĻ« +çŀ ħ +磷 éħ¸ +äºī åĪĽ +çİĭ æĻĵ +ç¼ĵ åĨ² +åİļ åİļ +åİļåİļ çļĦ +æŀ£ åºĦ +ç²¾ çĽĬ +ç²¾çĽĬ æ±Ĥ +ç²¾çĽĬæ±Ĥ ç²¾ +åĪĨæĶ¯ æľºæŀĦ +å®ŀæĸ½ ç»ĨåĪĻ +æĸ° èµĽåŃ£ +總 çµ± +éĢł è¡Ģ +é¢ĩ åħ· +é»Ħ åŁĶ +è¡Ģ èĦĤ +交éĢļ å·¥åħ· +å³ ¥ +æĹıèĩªæ²» å·ŀ +寺 éĻ¢ +確 å®ļ +æ¦Ĥ念 èĤ¡ +æĦŁ å®ĺ +æŁľ åı° +åĶ Ķ +çŀŃ解 並 +æĢ» ä»· +åIJ¸ åħ¥ +æĢ ¼ +æĻļ éĹ´ +å±Ĭ æ¯ķä¸ļçĶŁ +çĶŁ å§ľ +éĺħ读 åħ¨æĸĩ +å¾ĹåĪ° æľīæķĪ +æIJľ æķij +åİĨ æĿ¥ +èŃī æĺİ +åĥ » +èĨ³ é£Ł +åĦĦ åħĥ +æīĵ åİĭ +宾 客 +åķ ¼ +ä¸ĢçĻ¾ å¤ļ +æ·±åħ¥ 人å¿ĥ +æ¢ħ å·ŀ +çłĶ åѦ +åħ³ ä¹İ +è¼ Ľ +亲 åıĭ +éħį æĸĻ +æĪij çĪ±ä½ł +è´¸æĺĵ æĪĺ +æľī èī² +æľīèī² éĩijå±ŀ +æįIJ åĬ© +为 é¦ĸ +为é¦ĸ çļĦ +å¯Į åĬĽ +çĶ· ç¥ŀ +é³ ³ +æµĩ æ°´ +åIJ ± +æĺİç¡® æıIJåĩº +åı¹ äºĨ +åı¹äºĨ åı£æ°Ķ +礼 æĭľ +è¿Ļ个 åIJįåŃĹ +ä¿¡ å¾Ĵ +å¿Ĺ 强 +éĻIJ æŶ +æĶ¶ è²» +åĨľå®¶ ä¹IJ +å°ıé¾Ļ èĻ¾ +èIJ½ å¹ķ +æ§ Ł +åѦ 龸 +æĪĸ å¤ļ +æĪĸå¤ļ æĪĸ +æĪĸå¤ļæĪĸ å°ij +座è°Ī ä¼ļä¸Ĭ +æ¶ ¼ +éŃĶ çİĭ +å² ± +顶 å±Ĥ +顶å±Ĥ 设计 +èĦij åŃIJéĩĮ +éĻ¢ åŃIJéĩĮ +轩 è¾ķ +身å¿ĥ åģ¥åº· +èħ ij +éĹľ 注 +åıĤåĬł ä¼ļè®® +ä¸Ńåįİ æĸĩåĮĸ +追 寻 +å®ī çĦ¶ +é£Ļ åįĩ +éŁŃ èıľ +é¸ ¦ +åĤ¨ éĩı +çĶ· æĸ¹ +å¤ĩ 份 +æijĶ åĢĴ +润æ»ij æ²¹ +éĢ¼ è¿ij +çĶ³ è¯ī +鸣 ç±» +çŁ³æ²¹ åĮĸå·¥ +åĿļ æŀľ +è¿Ļ家 ä¼Ļ +æĭĴ ä¸į +羣 çļ® +è·Ŀ éĽ¢ +è¿ĺ æĮº +éĽķ åĥı +åĪĿ æģĭ +æıIJä¾Ľ æĽ´å¤ļ +æŁ¥çľĭ åħ¨æĸĩ +æķ°åŃĹ è´§å¸ģ +åĸī åĴĻ +åı¦ä¸Ģ ä½į +åĤ¬ åĮĸ +åĤ¬åĮĸ åīĤ +ä»İæĿ¥ 没 +å¯ĨåĪĩ 缸åħ³ +éĥ¨ 主任 +产åĵģ ç»ıçIJĨ +並 åIJĮæĦı +èIJ½ åħ¥ +å±ıå¹ķ ä¸Ĭ +åħ¬åı¸ ç«łç¨ĭ +æį¢ åı¥è¯Ŀ +æį¢åı¥è¯Ŀ 说 +ä½į æĸ¼ +ä½ Ķ +åĩ» æĿĢ +缸 è¾ĥ +缸è¾ĥ äºİ +ç²½ åŃIJ +åįĹ æŀģ +宫 é¢Ī +è£ģ åijĺ +æĺİ ç»Ĩ +ä»·åĢ¼ éĵ¾ +åĽĽä¸ª æĸ¹éĿ¢ +æĥħåĨµ æĿ¥çľĭ +æĮij åīĶ +æ® ĺ +æŀģ åĬĽ +çĸij éļ¾ +æĬµæĬĹ åĬĽ +æĢ¥ éĢŁ +æĪ Į +ä½İ ä¼° +éĹª è¿ĩ +æģ ¬ +èµŀ æī¬ +ä»ĸ å¦Ī +æĪIJ为 ä¸ĢåIJį +æ´Ĺ 礼 +é¢Ħ计 å°Ĩ +åħĪè¿Ľ åįķä½į +è¼ Ķ +éĢĥ èĦ± +çİ° åŃĺ +èĢģèĻİ æľº +åįģä¸ĥ æĿ¡ +åı¦ä¸Ģ åįĬ +温 æĥħ +åī¥ 离 +ä¸ĸ è´¸ +å®ĺ åı¸ +å¾Ī å·® +éĹ´ è·Ŀ +请 注æĦı +åı² è¯Ĺ +åĪ© åĻ¨ +è¿IJ ç®Ĺ +沦 为 +該 使çĶ¨èĢħ +èĮ ¬ +éĶ¦ 绣 +åı² æĸĻ +çģµ æ´»æĢ§ +èģĶ 社 +æĹł åĬ© +æĬĹ æ°§åĮĸ +èıľ èĤ´ +éĢł èĪ¹ +æİī èIJ½ +å¤į æŁ¥ +åĭĥ åĭĥ +åij¼ 声 +給 äºĪ +åIJĮäºĭ 们 +ç½ ° +è¯ķ æİ¢ +åħ³éĶ® åŃĹ +æįIJ çĮ® +ç»Łè®¡ æķ°æį® +åĪĽ ä½ľèĢħ +ä¸ĭ åįĬ +ä¸ĭåįĬ åľº +æī¿æĭħ 责任 +端 æŃ£ +ç©¿ è¡£ +ä¼ł çIJĥ +åĬ© éķ¿ +åĩ ± +éķ¶ åµĮ +é£ŀ ç¿Ķ +è¾ĵ åįµ +è¾ĵåįµ 管 +ä¸ĩ åħ¬éĩĮ +æݨ广 åºĶçĶ¨ +å¿« æ¨Ĥ +ç§ ½ +èī° å·¨ +åIJ¬ å®Į +åĿļ 硬 +奥 åľ° +å¥¥åľ° åĪ© +é¢ ĵ +èĻIJ å¾ħ +ä¾Ľ æ±Ĥ +éľī ç´ł +伪 è£ħ +乡 åľŁ +åĩ¡ æľ¬ç½ij +åĩ¡æľ¬ç½ij 注 +ä¼Ĭ åĪ© +è¡¡ æ°´ +æĽ´ åĥıæĺ¯ +åĪĨéĴŁ å·¦åı³ +è¦ı 模 +äºĶ åĪĨéĴŁ +åºĹ åĬłçĽŁ +åĽ° éĽ£ +åħ³ åģľ +æĢĿ 绪 +åĴ½ åĸī +缸 符 +çĥ¦ èºģ +æĻĤ æľŁ +åijĪ çı¾ +解 æķ£ +诱 导 +éļĶ çĥŃ +çĮ ¶ +åįĹ å®ĭ +æ·±åħ¥ äºĨ解 +çŃĶ çĸij +æĺ¼ å¤ľ +åįĥ ä¼ı +åĬ³åĬ¡ æ´¾éģ£ +红 è±Ĩ +åĿı äºĭ +çĤ¹ æ»´ +å°±ä¸ļ å²Ĺä½į +约 åIJĪ +åħį éĻ¤ +éĢĨ åĬ¿ +éĩį éĩijå±ŀ +å®ĺ 宣 +ä½İ å»ī +æģ¨ ä¸įå¾Ĺ +å¾Ĺ 天 +å¾Ĺ天 çĭ¬ +å¾Ĺ天çĭ¬ åİļ +ä¸Ģå°ģ ä¿¡ +æĬ½ å¥ĸ +è¾Ĺ 转 +çķĻ å®Ī +çķĻå®Ī åĦ¿ç«¥ +çŃĶ åį· +å·¨ åŀĭ +æľĢ好 ä¸įè¦ģ +æµĻæ±Ł 大åѦ +æĨ ¨ +æı¡ æīĭ +éĴĪ ç»ĩ +æİĴ 骨 +çĤ ½ +å°ģ è£ħ +åįĢ åŁŁ +空æ°Ķ åĩĢåĮĸ +åħī å½± +åĢĴ å¡Į +å§ļ æĺİ +æ¤į 被 +åѦ åīį +åѦåīį æķĻèĤ² +èĬĿ åĬł +èĬĿåĬł åĵ¥ +缩 æ°´ +ä½ Ł +åľ¨çº¿ åĴ¨è¯¢ +èµı æŀIJ +éĿĴ èĽĻ +æĬ± ä½ı +èĮĤ åIJį +åħ¨åĬĽ æīĵéĢł +åįļ士 åѦä½į +沧 å·ŀ +åĻ ¢ +æĿĤ çī© +åĪ» çĶ» +æį ħ +å¾® éĩı +å¾®éĩı åħĥç´ł +ä¸Ģ åĽŀäºĭ +鸡 èĤī +åĪ©æ¶¦ çİĩ +æīį ç®Ĺ +å¾® å¦Ļ +棵 æłij +è´ª 婪 +åĩı åĢ¼ +梦 å¢ĥ +åı¯ è§Ĩ +åı¯è§Ĩ åĮĸ +广大 å¸Ĥæ°ij +ä¸ĵä¸ļ ä»İäºĭ +ç»ı 纬 +ç´§ çĽ¯ +çŁ¥ å·± +è¤ ļ +æĸĩåĮĸ åºķèķ´ +åݦéŨ å¸Ĥ +临 港 +对åħ¶ 羣å®ŀ +岸 è¾¹ +è¦ĸ çĤº +æĬĹ çĻĮ +åĶIJ å®ĩ +ä¸įå¾Ĺ è¶ħè¿ĩ +å¨ģ æħij +æ¡Ĩæŀ¶ åįıè®® +èµ° ç§ģ +åĽ¢ å§Ķ +夸 大 +æ¬ Ħ +ç¥ŀç»ı ç³»ç»Ł +æijĦå½± ä½ľåĵģ +èĬ ¥ +å®ī åºĨ +æµ· 滨 +æŀĦ æĢĿ +çīµ æĮĤ +åı © +éĺIJ æĺİ +éģ ģ +ç²¾ æ²¹ +ç©´ ä½į +æĬ¤ 身 +æĬ¤èº« 符 +æĮĩ å°İ +åŃĺåľ¨ ä¸Ģå®ļ +å¯Ĥ éĿĻ +æµ·å¤ĸ å¸Ĥåľº +éĿ ¡ +综åIJĪ å¾ģ +ä¿ IJ +è¨Ī ç®Ĺ +æĺİ æľĹ +äºļ è¿IJ +äºļè¿IJ ä¼ļ +åīįçŀ» æĢ§ +åĮ® ä¹ı +产ä¸ļ æī¶è´« +èĦij æµ· +èĦijæµ· ä¸Ń +åħļçļĦ é¢Ĩ导 +åĪĺ éĤ¦ +æµģ æĺŁ +æĵ Ĥ +æĶĢ çĻ» +åĴ Ķ +ä¸Ģä¸ĭåŃIJ å°± +è¯Ĭ æ²» +使 åĬ² +åīµ ä½ľ +éĵŃ è®° +éĴ± è´¢ +æĹ¥æĬ¥ è®°èĢħ +çĥŁ çģ« +èĥľ è´Ł +åįļ 主 +ä¸ŃåĽ½ èģĶéĢļ +ç½ijç«Ļ é¦ĸ页 +å°± å¤Ł +å°±å¤Ł äºĨ +æīij åħĭ +å±ħ å§Ķä¼ļ +è° ¬ +å®īåħ¨ äºĭæķħ +åķĨ çĶ¨è½¦ +循çݯ ç»ıæµİ +æ· ¤ +èĢĥ è¯ģ +å®Ŀ èĹı +å®Į ç»ĵ +çłĶåıij æĬķåħ¥ +å² ij +æģŃ æķ¬ +离 éĢĢä¼ij +æ°´ 墨 +å© ¶ +è¯Ĺ åı¥ +å®ģæ³¢ å¸Ĥ +å¼± çĤ¹ +åģľ çīĮ +奶 æ²¹ +å¥ĩ纳 æ²³ +æĨ Ĥ +社ä¼ļ å®ŀè·µ +è´Ŀ 壳 +çłĤ æµĨ +èĪ¹ åıª +宣 æī¬ +综åIJĪ æķ´æ²» +åĤ ij +æ°ijæĹı æĸĩåĮĸ +éĩį çİ° +积 æ·Ģ +åħ¬ çĦ¶ +çħ ī +缸 èģļ +æ± ¾ +纹 çIJĨ +çĩĥ çħ¤ +æѤ ç§į +ç¾İ å¦Ĩ +åįĥ çĵ¦ +çIJ Ľ +驾驶 è¯ģ +éĺ¶ æ¢¯ +ä¸Ŀ ä¸Ŀ +å¾Īå¤ļ äºĭæĥħ +åħī éĺ´ +èijĹä½ľ æ¬Ĭ +åħ§ éĥ¨ +çĽ¸å¯¹ æĿ¥è¯´ +éĸ Ĵ +éľĩ æħij +說 話 +æĨ ij +ç«¥ è£ħ +ä½ıæĪ¿ åĴĮ +ä½ıæĪ¿åĴĮ åŁİ +å·²ç»ı è¶ħè¿ĩ +侦 å¯Ł +çŁ¿ çī© +ä¾Ľ 大家 +çī¹ éĤĢ +ç¨ĭåºı åijĺ +çķľçī§ ä¸ļ +æ° ª +çij ª +åĢĴ åľ¨ +åĢĴåľ¨ åľ° +æ¯ Ģ +梯 éĺŁ +æİ¥ èijĹ +æĬĹ èıĮ +è¤ ĩ +ç¬ Ļ +æ¯Ķ ä¸Ĭå¹´ +鸡 汤 +åŃ¦ä¹ł æĪIJ绩 +æĸij æĸĵ +åħĪ 导 +åĪĹ ä¸¾ +è°ĥæŁ¥ æĺ¾ç¤º +æ© « +ä¹Ŀ åįģ +è°¢ 飵 +è·¨è¶Ĭ å¼ı +女æĢ§ æľĭåıĭ +èIJ¥åħ» ä»·åĢ¼ +å®ŀè·µ ç»ıéªĮ +èĭı å·ŀå¸Ĥ +çĵ¶ åŃIJ +æĸ° çļĦä¸Ģ +æĸ°çļĦä¸Ģ å¹´ +æĺİ æĻ° +å®ł çĪ± +åŃŠ第 +æľĹ 诵 +纳 æĸ¯ +éĢĨ è¡Į +è«ĭ æĤ¨ +è«ĭæĤ¨ æıIJä¾Ľ +èĥ¸ æĢĢ +第ä¸ĥ å±Ĭ +强 壮 +代 åŃķ +汶 å·Ŀ +家 åĸ» +家åĸ» æĪ· +家åĸ»æĪ· æĻĵ +èħ ® +åIJ¯ 迪 +æĹł éļľç¢į +èĻķçIJĨ åıĬ +æĿ¥ åİĨ +å®ŀ åĬ¡ +ä¹Ł éļıä¹ĭ +æĬĢèĥ½ åŁ¹è®Ń +åѤ ç«ĭ +åī ģ +éĥ´ å·ŀ +æĶ¶ æķĽ +éł» éģĵ +èį£ 幸 +èİ« è¿ĩäºİ +æѤ æĻĤ +纪å§Ķ çĽij +纪å§ĶçĽij å§Ķ +缸 éĤ» +åı¦ä¸Ģ è¾¹ +çªĴ æģ¯ +æľīå¾Īå¤ļ ç§į +æ¯ı éĢ¢ +éĹ® ä¸ĸ +ç´¯ ç´¯ +éĿĴæĺ¥ æľŁ +è·¯ åĨµ +åħĭ èݱ +è¿Ħä»Ĭ 为æŃ¢ +æĥĬ å¥ĩ +è·¨ 度 +éħ¿ éĢł +åĩ ĭ +è¿ij ä¸īå¹´ +åĨħ 马 +åĨħ马 å°Ķ +æı į +è¿Ľå±ķ æĥħåĨµ +èĮ § +æľīåºı æİ¨è¿Ľ +æĢ» åĨłåĨĽ +æĪIJ绩 åįķ +éĽ»è©± åıĬ +ç´§å¯Ĩ ç»ĵåIJĪ +åºĬ ä½į +é¹ Ĭ +æķ£åıij çĿĢ +åĭŁ èµĦ +æ°¨ éħ¸ +彩 ç¥ŀ +è®Ģ åıĸ +éĩį 温 +ä¸Ń åŃĺåľ¨çļĦ +ç¾İ éºĹ +ä¸įæĸŃ å¢ŀåĬł +è½® æµģ +æİ¥ åIJ¬ +å¹´ 产åĢ¼ +åįĥ åħĭ +æĪĺåľº ä¸Ĭ +çħ§ 顧 +å¹²éĥ¨ éĺŁä¼į +åį° ç«ł +ä¸Ģèĩ´ æĢ§ +è¿ŀ å¤ľ +åħħ è£ķ +é»ij åIJįåįķ +åĩĢ æ°´ +ä¸Ģ大 æĹ© +åĮħ 袱 +çĬ¯ è§Ħ +çIJĨ è«ĸ +æŀģ æĺĵ +éª ¸ +å¨ĺ å¨ĺ +åĽ¢ åľĨ +亿åħĥ 以ä¸Ĭ +åĪ©çĶ¨ æĤ¨çļĦ +带æĿ¥ æĽ´å¤ļ +ä¸Ń央 空è°ĥ +æľĪ èĸª +çĮľ æĥ³ +åĪº 客 +ä½ľ æģ¯ +åįķ è°ĥ +äºĴ åĪ© +å¦Ĥæľī ä¾µæĿĥ +å°ı å·§ +åįģ åł° +åĵĪåĵĪ åĵĪåĵĪ +è¾¹ éĻħ +æłĩ è¯Ń +åĪĩåħ¥ çĤ¹ +éĢĨ è¢Ń +è¯ķ åīĤ +绿 è±Ĩ +è® ļ +åŁºçĿ£ å¾Ĵ +å£ ¬ +åħ¨ æĺİæĺŁ +éĢī ç§Ģ +èĪĮ å°ĸ +ä¸įåIJĮ ç±»åŀĭ +çĥŁ åĽ± +çģµ æ°Ķ +åĮº 管å§Ķä¼ļ +åĨľ åī¯ +åĨľåī¯ 产åĵģ +èĶļ æĿ¥ +沪 æĮĩ +åħ»æ®ĸ æĪ· +æĸĹ å¿Ĺ +é¦ĸ é¢Ĩ +è¡Ģ èħ¥ +åĬł ç´§ +ä¸Ģèĩ´ 好è¯Ħ +第ä¸ī èĬĤ +æī¬ å°ĺ +交éĢļ æŀ¢çº½ +鼶 ç¢İ +é»ij æ´ŀ +çľĭ ä¸įæĩĤ +å±ŀ å®ŀ +主 åŁİåĮº +å¨ Ľ +å¨Ľ æ¨Ĥ +ç¬ij æĦı +èĻ¹ æ¡¥ +åIJĦ个 çݯèĬĤ +çķ¥ å¾® +èĢķ èĢĺ +æľ¬ åľºæ¯ĶèµĽ +æĪIJ è´¥ +éĢī èĤ¡ +èªŀ è¨Ģ +çŃĶ 辩 +èĩª ä¹ł +æ£ º +ä¸ĩ 欧åħĥ +åģľ å·¥ +对åħ¶ è¿Ľè¡Į +积æŀģ éħįåIJĪ +ä¹¾ åĿ¤ +å¦ĸ æĢª +èļĮ åŁł +èµĦ产 è¯Ħä¼° +è°ĥ çļ® +éĻ¤ å¤ķ +åĽ´ å¢Ļ +æľį å½¹ +æ·± æ¸Ĭ +é¢Ħ åĪ¶ +ç ĥ½ +å®ī 稳 +建 æŀĦ +çĭĻ åĩ» +主åĭķ 註åĨĬ +éĥ½æľī èĩªå·± +æİĴåIJį 第ä¸Ģ +麻 è¾£ +çĢ ļ +çĥŁèĬ± çĪĨ +çĥŁèĬ±çĪĨ 竹 +èĩªçĦ¶ ä¿ĿæĬ¤ +ä»Ļ å¢ĥ +为äºĨ éģ¿åħį +åĨ· åºĵ +解æĶ¾ æĢĿæĥ³ +åĪĿ äºĮ +ä½ĵ è´´ +é¦ĸ å¯Į +迪 æĭľ +æļĤ ç¼ĵ +æĶ¯æĮģ åĬĽåº¦ +侦 æİ¢ +马 åĪº +åĮĹ æ±½ +ç¹ ŀ +è°İ è¨Ģ +éĢ£ çºĮ +å· ³ +ä»»ä½ķ æŶåĢĻ +车 èģĶç½ij +åįķ 项 +å¸Ń åį· +建çŃij æĿIJæĸĻ +ä¸Ńç§ĭ èĬĤ +ç¡ķ士 çłĶ究 +ç§ģ ç«ĭ +åħļåĴĮ æĶ¿åºľ +æľ¬æ¬¡ 交æĺĵ +èººåľ¨ åºĬä¸Ĭ +ç½ijåıĭ è¯Ħ论 +å¦ Ŀ +害 ç¾ŀ +åħ¬ç«ĭ åĮ»éĻ¢ +ä¸ ŀ +çĶŁçī© è´¨ +åºĶ éĤĢ +æĬ½ åıĸ +åĩł å¼ł +æijĺ ç¼ĸ +ç»ĺ æľ¬ +详 解 +强 硬 +æľĢ åħĪè¿ĽçļĦ +æĭĽ èĤ¡ +æĭĽèĤ¡ 书 +åįĥ æĸ¹ +åįĥæĸ¹ çĻ¾ +åįĥæĸ¹çĻ¾ 计 +éħį éŁ³ +驾 çħ§ +å¾ģ æĪĺ +èªĵ è¨Ģ +æĭľ å¸Ī +æĭľå¸Ī åѦ +æĭľå¸ĪåѦ èīº +æĬ± åĽ¢ +ç±³ ç²ī +éĿŀ常 éĢĤåIJĪ +èĪª æµ· +å±¥ 约 +åįģåħ« æĿ¡ +éĶ» éĢł +éĩįè¦ģ 举æݪ +åıijæĮ¥ ä½ľçĶ¨ +æ· ļ +人 社 +人社 å±Ģ +è¯ķçĤ¹ å·¥ä½ľ +éĺľ éĺ³ +æ¡ĥ åľĴ +æ°ij ä¼ģ +æ´ģ çĻ½ +è´µ 宾 +åħ¬ 社 +è§ī æĤŁ +è®°å¿Ĩ åĬĽ +æľĥåĵ¡ 註åĨĬ +æѤ æ¡Ī +麻 çĹ¹ +çı Ģ +æĸ© èİ· +çĶ· åŃ©åŃIJ +å±ĢéĻIJ äºİ +åĭĺ æŁ¥ +åIJĥ 饱 +èĬ¬ åħ° +æ£ķ èī² +ç¦ı ç¥ī +çĶ³ èĬ± +æµ· çĽĹ +èĶ ij +æĸĩ åѸ +æ´»æĢ§ çĤŃ +缴 éĢļ车 +è°¢ éĤĢ +躺 çĿĢ +åľ ĥ +æ¯ıæĹ¥ ç»ıæµİ +åħ¬åħ± æĸĩåĮĸ +讲 æķħäºĭ +å¯Ł çľĭ +æĤł éĹ² +åľ° åĿª +æ¶Į çİ°åĩº +é«ĺçŃī éĻ¢æł¡ +èĮĦ åŃIJ +éĺ² åį« +ä¾ĭ è¡Į +æĺ¾ éľ² +æĸ° 常æĢģ +ç»Ŀ ä½³ +å¯Į æ°ij +以 人æ°ij +以人æ°ij 为 +éĤ¢ åı° +å±ķ æ¼Ķ +çĻ¼ å¸ĥ +è´Ł è½½ +åģı 离 +æ°¸ éģł +éĩįè¦ģ åİŁåĽł +åįıä¼ļ ä¼ļåijĺ +éļ¾ æ°ij +çĶŁäº§ 车éĹ´ +çģµ åĬ¨ +两年 åīį +æĸ¹ åľĨ +æ´» ä¸ĭåİ» +ä¸ĸçķĮ è§Ĥ +éªĹ åıĸ +ç¾İ è²Į +èĥ½ çľĭåĩº +çĻ¼ æı® +è§Ĥ å½± +åī ĥ +åIJĪèµĦ åħ¬åı¸ +å© § +å¹² æű +åħŃ ä¸ªæľĪ +尤为 éĩįè¦ģ +èĤ ½ +秦 åĽ½ +æīĺ ç¦ı +建çŃij å¸Ī +åįĩ级 æĶ¹éĢł +å°ı é¢Ŀ +å°ıé¢Ŀ 贷款 +两个 ç»´æĬ¤ +æĭį æĭį +åı¯ çĸij +æį¢ åıĸ +æѦ 士 +èµĸ 以 +èµĸ以 çĶŁåŃĺ +æĮ ļ +殿 åłĤ +èĩªçĦ¶ çķĮ +ç£ģ åľº +å¦Ĥä½ķ çľĭå¾ħ +ä»ĬæĹ¥ 头æĿ¡ +西 åŁŁ +èİ· è¯Ħ +風 æł¼ +ä¿Ħ åĽ½ +æīĵ æĭ¼ +å®£ä¼ł çīĩ +å¾Ī æĸ¹ä¾¿ +ä¾Ľç»Ļ 侧 +纪念 ç¢ij +毫 åħĭ +èĬ³ é¦Ļ +å·¥åķĨ éĵ¶è¡Į +请 çĤ¹åĩ» +ç¼ ª +æĹłæķ° 次 +èį¯ å¸Ī +èħ ¸ +游 èīĩ +åĮ ¾ +å·¡ èĪª +æ²»çIJĨ ä½ĵç³» +èIJ¥éĢł èī¯å¥½ +æ·· æ·Ĩ +éĢļ çķħ +åĬ³ ç´¯ +ä»ĵ ä½į +å¢ŀ éķ· +éļIJ 约 +æĿĤå¿Ĺ 社 +åħ» èĤ² +åı¯èĥ½ åıijçĶŁ +èĢĥ 試 +西 侧 +åĬł åĢį +主æĮģ åı¬å¼Ģ +çķ¢ ç«Ł +éĹ® 询 +æµ· æ£ł +èĹ © +注æĺİ æĿ¥æºIJ +æ£Ģ çĸ« +请 åģĩ +æĬļ æij¸ +èĵĦ çĶµæ±ł +è·Ł ä¸įä¸Ĭ +çݰ代 社ä¼ļ +çŃ¹ èµĦ +ä½ĵèĤ² 彩票 +延 误 +è¾Ľ è¾£ +éĿ¢ 容 +åį° è®° +çģŃ äº¡ +ç´ł é£Ł +åħ´ èĩ´ +éľĢè¦ģ çĶ¨ +éľĢè¦ģçĶ¨ åĪ° +å®Ŀ å¦Ī +ç£ĭ åķĨ +éļ¶ å±ŀ +è´¡çĮ® åĬĽéĩı +åħ¬åħ± èµĦæºIJ +大 éĺª +åĨĽ è®Ń +æĤ¬ 念 +社ä¼ļ 稳å®ļ +å¹²äºĭ åĪĽä¸ļ +æľī æĿ¡ä»¶ +æľīæĿ¡ä»¶ çļĦ +ä¸Ģå¹´ ä¸Ģ度 +åİ ¥ +强 奸 +豪 车 +æİĮ æŁľ +æ°´åĪ© å·¥ç¨ĭ +å³ ª +积æŀģ ä½ľçĶ¨ +æµ· æ·Ģ +æµ·æ·Ģ åĮº +çĥŃ æĴŃ +åĿļæĮģ ä¸įæĩĪ +åıĮ èĦļ +绣 æĪĺ +ä»»ä½ķ 人éĥ½ +åľ°ä¸ĭ 室 +åĨ¶ çĤ¼ +è°ħ 解 +æ¸Ķ èĪ¹ +太éĺ³ åŁİ +被 æįķ +计ç®Ĺ åĻ¨ +西 åĮ» +èĪĴ å¿ĥ +æ¡ ¦ +éģ ² +åĬ ij +è¨ Ĺ +èİ º +åĸ ¬ +çĵ ¯ +åĺ ĺ +åł ķ +æķ Ŀ +åij ¦ +èĭ ŀ +æŃ ¹ +æĵ ¬ +æ£ Ħ +èĪ µ +å¥ ª +çļ ĭ +æĶ ¸ +åľ © +ç¤ Ļ +ç¢ ĺ +éı Ī +æĦ ķ +ç¹ ³ +èĺ ¸ +è² Ĥ +æ¼ ² +æij ¹ +æĶ Ŀ +åŃ ¢ +èķ Ń +é¨ ° +æ½ ¼ +éħ ° +æĴ ¥ +è¹ ¬ +é¨ Ļ +è¸ ¹ +éģ IJ +çĺ Ģ +èĽ ¤ +æĤ ĸ +çĴ ŀ +ç£ IJ +æİ ° +è¾ Ĭ +å¾ ij +æİ ĸ +éģ ŀ +éĤ ¸ +éĽ ı +æĨ İ +æľ ½ +çį » +ç® Ķ +è¤ ¶ +æļ ¢ +æĺ µ +çı Ĥ +æĤ ¸ +åģ µ +åĻ ľ +å£ ¯ +æĴ ® +æģ į +å© ķ +ç¯ ± +éĺ Ļ +çī ł +è£ ĺ +è³ ¢ +éĩ ľ +éĵ ł +èİ ĺ +æ® Ĩ +çĻ ¸ +è´ ı +ç² ± +å« ¡ +åĨ ¢ +è¤ Ĵ +æĩ Ĭ +éľ ĵ +å¡ µ +æĭ £ +å» Ł +é£ ½ +é¢ Į +åļ İ +æ· º +èĨ ł +åİ Ń +åļ ĩ +åij ĥ +çĴ ĭ +çŃ ± +æĭ · +èį § +éĶ ° +åŃ ° +èĵ ĵ +èĨ ½ +æŀ ī +åĸ ½ +çĽ Ķ +çŃ IJ +ç¾ ļ +è ħĮ +è¾ « +æ³ ĵ +çĶ ¬ +èŁ ² +åĸ ª +å¦ ĵ +è¬ Ģ +çĤ Ĭ +æĽ ľ +æ± IJ +è´ Ī +èį Ģ +æĬ ł +ç¢ ¾ +æ« ĥ +éŀ ł +èij Ĩ +ç¥ ¯ +å½ Ŀ +é¦ į +åĮ £ +æľ Ń +åĿ Ĥ +ä¿ ij +èĵ ® +çij Ľ +æī ī +èĩ Ł +è² « +çİ ¥ +æ· ¼ +åİ ² +é³ Į +å³ Ń +åij Ľ +é § +é§ IJ +éģ · +ä¿ ª +æĢ Ĥ +è¾ į +å± į +åĭ ģ +å¥ ļ +éļ ħ +éĴ ´ +è¼ Ŀ +å® ¦ +èIJ ĥ +çĺ ĭ +æĨ ¶ +æĤ ħ +è¾ Ļ +åij ľ +çł º +éĢ ŀ +æµ ļ +éĸ £ +èĸ © +éĻ ĭ +çĤ Ļ +èª ķ +ä¸ Ł +é¹ ½ +ç± Į +è´ ° +éĭ ª +çľ © +æĴ IJ +èĨ º +éŀ ĺ +ç¾ ² +çª ® +ç´ IJ +æ® ´ +çº ¾ +èº į +ç´ ĭ +çĦ ĸ +çĶ º +çī ½ +çĤ ¯ +ç¼ Ķ +æ¯ ĵ +å¬ ° +æ¢ § +äº Ł +è¢ ħ +çį Ħ +è¿ ¥ +æ¼ ¾ +çĿ ij +ç¸ ¾ +é¦ ĭ +é¤ ħ +æ ¹Ħ +æĺ ĩ +æŀ Ń +èĸ ° +æŁ ij +æ¦ » +åĻ Ĺ +åĻ ´ +æ£ £ +åĶ § +çĨ ¹ +è¼ ¯ +å¢ Ł +é² ² +æĪ Ľ +èī ¦ +èĬ ® +åĺ Ł +å¸ ¥ +å¿ » +çĮ Ŀ +å¯ µ +è³ ¦ +èĽ ¾ +æ» ¾ +çĤ ķ +éĵ ¬ +èĴ ¿ +éĴ ¨ +çĥ Ļ +ç² ķ +æĥ ¦ +æº § +é¢ į +éħ £ +å³ ¦ +ç± ģ +çĥ ĥ +åĨ Ĺ +åı ģ +çĽ § +ç½ µ +éĴ Ĺ +å¬ ī +è° ı +ç³ § +è¾ Ń +æ· ¬ +èŁ Ĵ +è¯ © +è¦ ĥ +çĻ ĸ +é½ Ĵ +çĪ IJ +ç® į +ç¼ İ +ç£ º +è¯ « +è¤ ² +æĵ ł +èIJ ¦ +çĿ ¬ +è° į +éĦ ° +æł ¾ +é¡ ı +ç¸ ± +æ¡ ¨ +éĨ ¬ +è¥ ² +è® ª +å© º +èį Ł +åĮ Ŀ +çĨ ł +èĽ Ĭ +æ¸ ļ +å´ ½ +é² ¤ +åķ ° +åĮ ķ +ä¸ IJ +è® ¥ +åı ½ +åı ¼ +çļ ¿ +è¿ Ĥ +åIJ Ĩ +å± ¹ +èĩ ¼ +è® ¹ +é© ® +çº « +æ± ŀ +æĬ ¡ +èĭ ĩ +åIJ ł +åIJ Ń +åIJ ® +å² ĸ +ä½ ĥ +çĭ Ī +åº ĩ +åIJ Ŀ +éĹ ° +æ± ¹ +å¿ ± +æĭ Ħ +æĭ Ĺ +èĮ ī +èĭ Ľ +èĮ ģ +çŁ ¾ +èĻ ı +åij » +åĴ Ħ +å¿ ¿ +èĤ ® +çĭ ŀ +çĸ Ł +çĸ Ļ +çĸ ļ +æ³ ŀ +å¸ ļ +å± ī +è¿ ¢ +é© ¹ +ç İ· +çıĬ ó +çıĬó ł +çıĬół Ħ +çıĬółĦ ģ +æĮ İ +æĭ ´ +åŀ Ľ +èį ¤ +æ® ĥ +çĽ ¹ +åĵ Ĩ +è´ » +æ¯ ¡ +çĭ ° +çĭ ¡ +æŁ Ĵ +æģ ĥ +è¯ ¬ +è¢ Ħ +è¯ ² +èļ ¤ +èĢ Ļ +åŁ Ĥ +æį İ +æį Į +æ¢ Ĩ +é ħĮ +çł ¾ +æ® ī +åĶ ł +æĻ Į +èļ £ +èļ ª +èļ ĵ +é¸ ¯ +åĶ ģ +åĶ Ĩ +åĢ Ķ +èĪ Ģ +è± º +èĥ ° +é¸ µ +é¸ ³ +é¦ ģ +ç¾ Ķ +æ¶ £ +æ¶ ķ +æĤ ¯ +è¯ ½ +è° Ĩ +ç¥ Ł +ç» ¢ +æį º +æį ¶ +æį » +æİ Ĥ +èı ł +èIJ ¤ +éħ Ĺ +çľ ¶ +åķ Ħ +èļ ¯ +èĽ Ģ +åĶ ¬ +å¸ · +éĵ IJ +éĵ Ľ +åģ İ +å¾ Ļ +èĦ ¯ +è± ļ +çĮ ĸ +çĹ Ĭ +æ¶ ® +æĥ Ń +æĤ ´ +æĥ ĭ +è° ļ +æı © +æIJ Ģ +æIJ Ķ +æ¦ Ķ +æ¤ Ń +éĽ ³ +åĸ ³ +è· Ľ +èľ ĵ +èľ Ĵ +é¹ ĥ +éĶ Ħ +çĶ ¥ +çŃ ı +çĮ © +çĮ ¬ +çĮ ¾ +çĹ ¢ +çĹ ª +æĥ ° +çª ĺ +è° ¤ +éļ ĺ +å© ¿ +é¹ ī +çij Ļ +æĸ Ł +æ¤ ¿ +éħ ª +éĽ ¹ +åĹ ¦ +è· · +è· º +è· ¤ +èľ Ī +èľ Ĺ +å¹ Į +é¦ ı +èª Ĭ +æ¼ ĵ +è¤ Ĥ +èĶ Ĺ +èĶ ¼ +åħ ¢ +è£ ³ +èľ » +èĿ ĩ +åĺ Ģ +éĶ ¹ +ç® ķ +ç® © +çĺ © +çĺ Ł +æ¼ ± +å¯ ¥ +éª ¡ +æĴ µ +æĴ ¬ +è± Į +åĺ ¹ +èĿ ł +èĿ Į +èĿ Ĺ +èĿ Ļ +éķ IJ +ç¨ ¼ +ç¯ ĵ +èĨ Ľ +é² « +çĺ ª +é² ¨ +æĨ Ķ +ç¿ © +è¤ ¥ +ç¼ Ń +åĻ © +çĵ ¢ +éľ İ +è¸ ± +è¹ Ĥ +èŁ Ĩ +é¹ ¦ +ç¯ ¡ +çĺ ¸ +çª ¿ +ç¼ ° +èĹ IJ +è¹ ĭ +èŁ ĭ +èŁ Ģ +èµ ¡ +èĩ Ĭ +é³ Ħ +ç³ ł +æĩ ¦ +åļ £ +éķ ° +é³ į +ç° ¸ +çĻ £ +é³ ĸ +é¬ ĵ +èł ķ +éľ ¹ +èº ı +é» ¯ +çĵ ¤ +çŁ Ĺ +ä¹ Ĥ +ä¹ ľ +åħ Ģ +å¼ ĭ +åŃ ij +åŃ ĵ +å¹ º +äº ĵ +å »¿ +ä¸ ı +åį ħ +ä» ĥ +ä» ī +ä» Ĥ +åĪ Ī +çĪ » +åį ŀ +éĹ © +è® £ +å¤ ¬ +çĪ ¿ +æ¯ ĭ +éĤ Ĺ +éĤ Ľ +èī ½ +èī ¿ +åı µ +ä¸ ķ +åĮ ľ +åĬ ¢ +åį Ł +åı ± +åı » +ä» ¨ +ä» Ł +ä» ¡ +ä» « +ä» ŀ +åį ® +æ° IJ +çĬ ° +åĪ į +éĤ Ŀ +éĤ Ļ +è® ¦ +è® § +è® « +å° » +éĺ ¡ +å° ķ +å¼ ģ +èĢ Ĵ +çİ İ +çİ ij +åľ ¬ +æī ¦ +åľ ª +åľ ¹ +æī ª +åľ ® +åľ ¯ +èĬ Ĭ +èĬ į +èĬ Ħ +èĬ ¨ +èĬ ij +èĬ İ +èĬ Ĺ +äº ĺ +åİ į +å¤ ¼ +æĪ į +å° ¥ +ä¹ © +æĹ ¯ +æĽ ³ +å² Į +å± º +åĩ ¼ +åĽ ¡ +éĴ ĩ +ç¼ ¶ +æ° ĺ +æ° ĸ +çī Ŀ +ä¼ İ +ä¼ Ľ +ä¼ ¢ +ä½ ¤ +ä» µ +ä¼ ¥ +ä¼ § +ä¼ ī +ä¼ « +åĽ Ł +æ± Ĩ +åĪ ĸ +å¤ Ļ +æĹ ® +åĪ İ +çĬ · +çĬ ¸ +èĪ Ľ +åĩ « +é Ĥ¬ +é¥ § +æ± Ķ +æ± ľ +æ± Ĭ +å¿ ĸ +å¿ ı +è® ´ +è® µ +è® · +èģ ¿ +èī ® +åİ ¾ +å¦ ģ +çº ¡ +çº £ +çº ¥ +çº ¨ +çİ ķ +çİ Ļ +æĬ Ł +æĬ Ķ +åľ » +åĿ į +æĬ ĥ +㧠IJ +èĬ « +èĬ ¾ +èĭ Ī +èĭ £ +èĭ ĭ +èĬ ¼ +èĭ Į +èĭ ģ +èĬ © +èĬ ª +èĬ ¡ +èĬ Ł +èĭ Ħ +èĭ İ +èĭ ¡ +æĿ Į +æĿ ĵ +æĿ Ī +å¿ ij +åŃ Ľ +éĤ ´ +éĤ ³ +å¥ ģ +è± ķ +å¿ Ĵ +æ¬ ¤ +è½ « +è¿ ĵ +éĤ ¶ +å¿ IJ +åį £ +éĤ º +æĹ ° +åij ĭ +åij Ĵ +åij ĵ +åij Ķ +åij ĸ +æĹ ¸ +åIJ ¡ +èĻ ¬ +åIJ ½ +åIJ £ +åIJ ² +å¸ ı +å² Ī +å² ĺ +åħ ķ +åĽ µ +åĽ « +éĴ Ĭ +éĴ ĭ +é ĴĮ +è¿ ķ +æ° Ļ +æ° ļ +çī ¤ +ä½ ŀ +ä½ ļ +ä½ Ŀ +ä½ Ĺ +å½ · +ä½ ĺ +ä½ ¥ +è± ¸ +åĿ Į +èĤ Ł +å¥ Ĥ +åĬ ¬ +çĭ ģ +é¸ ł +é¥ ¨ +é¥ © +é¥ « +é¥ ¬ +åº ij +åº ĭ +çĸ Ķ +çĸ ĸ +èĤ ĵ +éĹ ± +éĹ ³ +çĤ Ģ +æ² £ +æ² ħ +æ² Ķ +æ² ¤ +æ² ı +æ² ļ +æ± © +æ± ¨ +æ² ¨ +æ± ´ +æ² Ĩ +æ² © +æ³ IJ +æĢ ĥ +æĢ Ħ +å¿ ¡ +å¿ ¤ +å¿ ¾ +æĢ ħ +å¿ ª +æĢ Ĩ +å¿ Ń +å¿ ¸ +è¯ Ĥ +è¯ ĥ +è¯ ħ +è¯ ĭ +è¯ Į +è¯ Ĵ +éĻ Ĥ +éĻ ī +å¦ © +å¦ ª +å¦ £ +å¦ Ĺ +å¦ « +å§ Ĵ +å¦ ¤ +åĬ Ń +åĪ Ń +éĤ ° +çº Ń +çº ° +çº ´ +çİ ¡ +çİ Ń +çİ ł +çİ ¢ +çİ ¦ +çĽ Ĥ +å¿ Ŀ +åĮ ¦ +åĿ © +æĬ ¨ +æĭ ¤ +åĿ « +æĭ Ī +åŀ Ĩ +æĬ » +åĬ ¼ +æĭ ĥ +æĭ Ĭ +åĿ ¼ +åĿ » +㧠Ł +åĿ ¨ +åĿ Ń +æĬ ¿ +åĿ ³ +èĭ · +èĭ ¤ +èĮ ı +èĭ « +èĭ ľ +èĭ ´ +èĭ Ĵ +èĭ ĺ +èĮ Į +èĭ » +èĭ ĵ +èĮ ļ +èĮ Ĩ +èĮ ij +èĮ ĵ +èĮ Ķ +èĮ ķ +è ĮĢ +èĭ ķ +æŀ ¥ +æŀ ĩ +æĿ ª +æĿ ³ +æŀ § +æĿ µ +æŀ ¨ +æŀ ŀ +æŀ ĭ +æĿ » +æĿ · +æĿ ¼ +çŁ ¸ +ç łĢ +åĪ ³ +å¥ Ħ +æ® ģ +éĥ ı +è½ Ń +éĥ ħ +é¸ ¢ +çĽ ± +æĺ Ļ +æĿ ² +æĺ ĥ +åĴ Ĥ +åij ¸ +æĺ Ģ +æĹ » +æĺ ī +çĤ ħ +çķ Ģ +èĻ ® +åĴ Ģ +åij · +é» ¾ +åij ± +åij ¤ +åĴ Ĩ +åĴ Ľ +åij ¶ +åij £ +åĴ Ŀ +å² ¢ +å² ¿ +å² ¬ +å² « +å¸ Ļ +å² £ +å³ ģ +åĪ ¿ +å² · +åī Ģ +å¸ Ķ +å³ Ħ +æ² ĵ +åĽ ¹ +ç½ Ķ +éĴ į +éĴ İ +éĴ ı +éĴ Ĵ +éĴ ķ +éĤ ¾ +è¿ ® +çī ¦ +ç« º +è¿ ¤ +ä½ ¶ +ä¾ ij +ä¾ ī +èĩ ¾ +ä¾ Ĺ +ä¾ ı +ä¾ © +ä½ » +ä½ ¾ +ä¾ ª +ä½ ¼ +ä½ ¯ +ä¾ ¬ +å¸ Ľ +ä¾ Ķ +å¾ Ĥ +åĪ ½ +éĥ Ħ +ç± ´ +çĵ ® +æĪ Ĺ +èĤ ¼ +äı Ŀ +èĤ ± +èĤ « +è¿ © +éĥ ĩ +çĭ İ +çĭ į +çĭ Ĵ +åĴ İ +é¥ ¯ +é¥ ´ +åĨ ½ +åĨ ¼ +åº ĸ +çĸ ł +çĸ Ŀ +åħ ĸ +åĬ ¾ +ð¬ ī +ð¬ī ¼ +çĤ ĺ +çĤ Ŀ +çĤ Ķ +æ³ Ķ +æ² Ń +æ³ · +æ³ ± +æ³ ħ +æ³ ł +æ³ º +æ³ ĸ +æ³ « +æ³ ® +æ² ± +æ³ ¯ +æĢ Ļ +æĢ µ +æĢ ¦ +æĢ Ľ +æĢ ı +æĢ į +ã ¤ +㤠ĺ +æĢ © +æĢ « +æĢ ¿ +å® ķ +ç© ¹ +å® ĵ +è¯ ĵ +è¯ Ķ +è¯ ĸ +è¯ ĺ +æĪ ¾ +è¯ Ļ +æĪ ½ +éĥ ĵ +è¡ © +ç¥ Ĩ +ç¥ İ +ç¥ ĩ +è¯ ľ +è¯ Ł +è¯ £ +è¯ ¤ +è¯ § +è¯ ¨ +æĪ ķ +éĻ Ķ +å¦ ² +å¦ ¯ +å§ Ĺ +å¸ ij +åŃ ¥ +é© ½ +èĻ ± +è¿ ¨ +ç» Ģ +ç» ģ +ç» Ĥ +é© · +é© ¸ +ç» ī +ç» Į +éª Ģ +çĶ ¾ +çı ı +çı IJ +çı ij +çİ ³ +é¡ ¸ +çı ī +çı Ī +æĭ ® +åŀ Ń +æĮ Ŀ +æĮ ŀ +åŀ ¤ +èµ ³ +è´ ² +åŀ ± +åŀ Į +åŀ § +åŀ ĵ +æĮ ¦ +åŀ ł +èį ļ +èį ij +è´ ³ +èį ľ +èİ Ĵ +èĮ ¼ +èĮ ´ +èĮ ± +èİ Ľ +èį ŀ +èĮ ¯ +èį ı +èį ĩ +èį ĥ +èį ł +èĮ Ń +åŀ © +èį ¥ +èį ¦ +èį ¨ +èį © +åī ĭ +èį ª +èį ¬ +èį ® +æŁ ° +æł ī +æŁ ĺ +æł Ĭ +æŁ © +æŀ ° +æł Į +æŁ Ļ +æŀ µ +æŀ ³ +æŁ ŀ +æŁ Ŀ +æł Ģ +æŁ ¢ +æł İ +æŁ Ī +æŁ ģ +æŀ · +æŁ ½ +åī Į +éħ Ĭ +éĥ ¦ +çĶ Ń +çł Ĺ +çł ĺ +çł Ĵ +æĸ « +çł Ń +çł ľ +èĢ · +èĻ º +æ® Ĥ +æ® ĩ +æ® Ħ +è½ ± +è½ ² +è½ ³ +è½ ¶ +è½ ¸ +èĻ ¿ +æ¯ ĸ +è§ ĩ +å° ľ +åĵ IJ +çľ Ħ +çľ į +ðł ³ +ðł³ IJ +éĥ ¢ +çľ ĩ +çľ Ĭ +çľ Ī +ç¦ º +åĵ Ĥ +åĴ ´ +æĽ · +æĺ ´ +åĴ ¦ +åĵ ĵ +åĵ Ķ +çķ İ +åij ² +èĥ Ħ +çķ ĭ +çķ Ī +èĻ ¼ +èĻ » +çĽ ħ +åĴ £ +åĵ ķ +åī IJ +éĥ § +åĴ » +åĽ ¿ +åĴ ¿ +åĵ Į +åĵ Ļ +åĵ ļ +åĴ © +åĴ ¤ +åĵ Ŀ +åĵ ı +åĵ ŀ +å³ £ +ç½ ĺ +å³ Ĵ +å³ ¤ +å³ ĭ +è´ ¶ +éĴ ļ +éĴ ¡ +éĴ £ +éĴ ¤ +éĴ « +æ° ¡ +çī ¯ +éĥ ľ +ç§ ķ +ç§ Ń +ç« ½ +ç¬ Ī +ä¿ ¦ +ä¿ ¨ +ä¿ ħ +åı Ł +åŀ ¡ +çī ® +ä¿ £ +ä¿ ļ +çļ Ī +ä¿ Ł +éĢ ħ +å¾ ĩ +å¾ ī +èĪ ¢ +éĥ Ĺ +ä¿ İ +éĥ ¤ +çĪ ° +éĥ Ľ +çĵ ´ +èĥ ¨ +èĥ ª +èĥ Ľ +èĥ Ĥ +èĥ Ļ +èĥ į +èĥ Ĺ +è ĥĿ +æľ IJ +èĥ « +é¸ ¨ +åĮ į +çĭ ¨ +çĭ ¯ +é£ ij +çĭ © +çĭ ² +è¨ ĩ +éĢ Ħ +æĺ Ŀ +é¥ · +é¥ ¸ +é¥ ¹ +åŃ ª +å¨ Ī +åº ¥ +çĸ ¬ +çĸ £ +çĸ ¥ +çĸ Ń +åº ł +ç« ij +é£ Ĵ +éĹ ¼ +éĹ ¾ +éĹ ¿ +éĺ Ĥ +ç¾ ij +è¿ ¸ +ç± ¼ +éħ ĭ +çĤ » +çĥ Ģ +çĤ · +æ´ ± +æ´ ¹ +æ´ § +æ´ Į +æµ ĥ +æ´ ĩ +æ´ Ħ +æ´ Ļ +æ¶ İ +æ´ İ +æ´ « +æµ į +æ´ ® +æ´ µ +æµ Ĵ +æµ Ķ +æµ ķ +æ´ ³ +æģ ¸ +æģ ĵ +æģ ¹ +æģ « +æģ » +æģ Ĥ +æģ ª +æģ ½ +å® ¥ +æī ĥ +è¡ ² +è¡ ½ +è¡ ¿ +è¢ Ĥ +ç¥ ľ +ç¥ ĵ +ç¥ ļ +è¯ ® +ç¥ Ĺ +ç¥ ¢ +è¯ ° +è¯ ³ +é¸ © +æĺ ¶ +åĴ « +å¼ Ń +çī ģ +èĥ ¥ +éĻ Ł +å§ ® +å¨ Ĩ +å§ Ŀ +å§ £ +å§ ĺ +å§ ¹ +ç¾ ¿ +çĤ ± +çŁ ľ +ç» Ķ +éª ģ +éª ħ +ç» Ĺ +ç» Ľ +éª Ī +èĢ ĸ +æĮ Ī +çı ¥ +çı Ļ +é¡ ¼ +çı ° +çı © +çı § +çı £ +çı ŀ +çIJ ¤ +çı ² +æģ ļ +åŁ ķ +åŁ ĺ +åŁ Ļ +åŁ ļ +æĮ ¹ +èĢ Ĩ +èĢ Ħ +åŁ Ĵ +æį ĭ +è´ ½ +åŀ ¸ +æį ĥ +çĽ į +èį ¸ +èİ ³ +èİ ´ +èİ ª +èİ ł +èİ ľ +èİ ħ +èį ¼ +èİ © +èį ½ +èİ ¸ +èį » +èİ ¨ +é¸ ª +èİ ¼ +æł ² +æł ³ +æ¡ ¡ +æ¡ İ +æ¡ ¢ +æ¡ ¤ +æ¢ ĥ +æł Ŀ +æ¡ ķ +æ¡ ģ +æ¡ § +æ¡ ħ +æł Ł +æ¡ ī +æł © +éĢ ij +éĢ ĭ +å½ § +é¬ ² +è± ĩ +éħ IJ +éĢ ¦ +åİ Ŀ +åŃ ¬ +çł Ŀ +çł ¹ +çł § +çł · +çł Ł +çł ¼ +çł ¥ +çł £ +åī ŀ +çł » +è½ ¼ +è½ ¾ +è¾ Ĥ +é¸ « +è¶ ¸ +é¾ Ģ +é¸ ¬ +èĻ Ķ +çľ ¬ +åĶ Ľ +çľ Ļ +åĵ § +åĵ ½ +æĻ ģ +é¸ ® +è¶ µ +è¶ ¿ +çķ Ľ +èļ ¨ +èļ ľ +èļ į +èļ ĭ +èļ ¬ +èļ Ŀ +èļ § +åĶ ¢ +åľ Ħ +åĶ £ +åĶ ı +çĽ İ +åĶ ij +å´ Ĥ +å´ ĥ +ç½ ¡ +ç½ Ł +è§ Ĭ +èµ ħ +éĴ ² +éĴ µ +éĴ ¹ +éĴ º +éĴ ½ +éĴ ¼ +éĴ ¿ +éĵ Ģ +éĵ Ħ +éĵ Ĩ +éĵ Ī +éĵ ī +éĵ Ĭ +éĵ ĭ +éĵ Į +é ĵį +ä ¥ +ä¥ ½ +éĵ İ +æ° © +æ° ¤ +æ° ¦ +æ¯ ª +èĪ IJ +ç§ £ +ç§ « +çĽ ī +ç¬ Ħ +ç¬ ķ +ç¬ Ĭ +ç¬ ı +ç¬ Ĩ +ä¿ ¸ +ä¿ µ +åģ Į +ä¿ ³ +ä¿ ¶ +åĢ ¬ +åĢ ı +æģ ģ +åĢ Ń +ä¿ ¾ +åĢ ľ +éļ ¼ +éļ ½ +åĢ Į +åĢ ¥ +èĩ ¬ +éĥ « +åĢ ¨ +è¡ Ħ +é¢ Ģ +å¾ ķ +èĪ « +è¡ ¾ +èĥ ¯ +èĥ ± +èĥ ´ +èĥ Ń +èĦ į +èĥ ¼ +èĦ Ĵ +é¸ ± +é¸ ² +çĭ · +çĮ ģ +çĭ ³ +çĮ ĥ +çĭ º +éĢ ĸ +æ¡ Ģ +é¥ ½ +åĩ ĩ +æĮ Ľ +äº ³ +çĸ ³ +çĸ ´ +çĸ ¸ +çĸ ½ +çĹ Ī +çĸ ± +çĹ Ĥ +çĹ ī +è¡ ® +é¢ ĥ +æģ £ +æĹ Ĩ +æĹ Ħ +æĹ ĥ +éĺ ĥ +éĺ Ħ +è¨ ļ +éĺ Ĩ +æģ Ļ +ç² ij +çĥ ľ +çĥ © +çĥ Ĭ +åī ¡ +éĥ ¯ +çĥ ¬ +æ¶ ij +æµ ¯ +æ¶ ŀ +æ¶ Ł +å¨ ij +æ¶ ł +æµ ŀ +æ¶ ĵ +æµ ¥ +æ¶ Ķ +æµ ľ +æµ ł +æµ £ +æĤ ļ +æ ĤŃ +æĤ Ŀ +æĤ Ĵ +æĤ Į +æĤ Ľ +çª Ī +åī ľ +è¯ ¹ +è¯ ¼ +è¢ Ĵ +è¢ ¢ +è¯ ¿ +è° Ģ +è° Ĥ +è° Ħ +è° ĩ +å± IJ +å± Ļ +éĻ ¬ +åĭ IJ +å¥ ĺ +çī Ĥ +èļ © +éĻ ² +å¨ Į +å¨ ī +å¨ ² +å¨ ´ +å¨ £ +å¨ ĵ +å© Ģ +çķ ļ +éĢ ¡ +ç» ł +éª Ĭ +ç» ¡ +éª ĭ +ç» ¦ +ç» ¨ +éª İ +éĤ ķ +é¸ ¶ +å½ Ĺ +èĢ ľ +çĦ ĺ +èĪ Ĥ +çIJ ı +çIJ ĩ +éº ¸ +æı ¶ +åŁ ´ +åŁ ¯ +æį ¯ +æİ ³ +æİ ´ +åŁ ¸ +åŁ µ +èµ § +åŁ ¤ +æį Ń +éĢ µ +åŁ Ŀ +åł ĭ +åł į +æİ ¬ +é¸ · +æį ½ +æİ Ĭ +åł ī +æİ ¸ +æį © +æİ ® +æĤ « +åŁ Ń +åŁ ½ +æİ ĩ +æİ ¼ +èģ ĥ +èIJ ģ +èı ĺ +åł ĩ +èIJ ĺ +èIJ ĭ +èı ½ +èı ĸ +è IJľ +èIJ ¸ +èIJ ij +æ£ » +èı Ķ +èı Ł +èIJ ı +èı ¹ +èı ª +èı ħ +èı Ģ +èı ° +èı ¡ +æ¢ ¿ +æ¢ ı +è§ ĭ +æ¡ ´ +æ¡ · +æ£ ģ +æ¡ « +æ£ Ĥ +åķ ¬ +éĥ ¾ +æķ ķ +è± ī +éĦ Ħ +éħ ŀ +ç¡ İ +ç¡ Ń +ç¡ ĸ +ç¡ Ĺ +ç¡ IJ +ç¡ ĩ +ç¡ Į +é¸ ¸ +çĵ ł +åĮ ı +åİ © +æ® Ĵ +æ® ĵ +æ® į +èµ ī +éĽ © +è¾ Ħ +åł ij +çľ Ń +çľ ¦ +åķ § +æĻ ¡ +æĻ ¤ +çľ µ +åľ Ĭ +åĸ ı +åķ ī +åĭ ĸ +æĻ ŀ +åĶ µ +æĻ Ĺ +åķ Ń +çķ ¦ +è¶ º +åķ ® +è· Ħ +èļ ¶ +è ĽĦ +èĽ İ +èĽ Ĩ +èļ ° +åľ ī +èļ ± +èĽ ī +èĽ ı +èļ ´ +åķ ģ +åķ ķ +åĶ ¿ +åķ IJ +åĶ ¼ +åĶ · +åķ ĸ +åķ µ +åķ ¶ +åķ · +åĶ ³ +åĶ ° +åķ ľ +å¸ » +å´ ļ +å´ ¦ +å¸ ¼ +å´ ® +å´ ¤ +å´ Ĩ +èµ ĩ +èµ Ī +èµ Ĭ +éĵ ij +éĵ Ĵ +éĵ Ĺ +éĵ Ļ +éĵ Ł +éĵ ¡ +éĵ ¢ +éĵ £ +éĵ ¤ +éĵ § +éĵ ¨ +éĵ © +éĵ ª +éĵ « +éĵ ¯ +éĵ ° +éĵ ± +éĵ ³ +éĵ µ +éĵ · +çī ¾ +é¸ ¹ +ç§ ¾ +éĢ ¶ +ç¬ º +çŃ ĩ +ç¬ ¸ +ç¬ ª +ç¬ ® +ç¬ ł +ç¬ ¥ +ç¬ ¤ +ç¬ ³ +ç¬ ¾ +ç¬ ŀ +åģ ¾ +åģ ĥ +åģ ķ +åģ Ī +åĤ Ģ +åģ ¬ +åģ » +çļ ij +çļ İ +é¸ » +å¾ ľ +èĪ ¸ +èĪ » +èĪ ´ +èĪ · +é¾ Ľ +ç¿ İ +èĦ ¬ +èĦ ĺ +èĦ ² +åĮ IJ +çĮ Ĺ +çĮ ¡ +çĮ ŀ +æĸ Ľ +çĮ ķ +é¦ Ĺ +é¦ ĥ +é¦ Ħ +é¸ ¾ +åº ¹ +åº ¾ +çĹ Ķ +çĹ į +ç¿ Ĭ +æĹ Į +æĹ İ +è¢ ¤ +éĺ ĩ +éĺ Ī +éĺ ī +éĺ Ĭ +éĺ ĭ +éĺ į +éĺ ı +ç¾ Ł +ç² Ŀ +çĦ IJ +çĦ ĵ +çĦ Ĺ +æ· ħ +æ· ŀ +æ¸ İ +æ¶ ¿ +æ· ĸ +æĮ ² +æ· ł +æ¶ ¸ +æ¸ ij +æ· ¦ +æ· Ŀ +æ¶ ª +æ· Ļ +æ¶ « +æ¸ Į +æĤ » +æĤ ± +æ ĥĿ +æĥ ĺ +æĥ Ĩ +æĥ ļ +æĥ ĩ +æĥ ® +çª ķ +è° Į +æī Ī +çļ ² +è° ij +è£ Ĩ +è¢ · +è£ ī +è° Ĵ +è° Ķ +è° ķ +è° ĸ +è° Ĺ +è° Ļ +è° Ŀ +éĢ ¯ +éĥ ¿ +éļ Ī +ç² ľ +éļ į +éļ Ĺ +å© Ĭ +å¨ ¼ +å© ¢ +å© µ +èĥ ¬ +è¢ Ī +ç¿ Į +æģ ¿ +æ¬ ¸ +ç» « +éª IJ +ç» ¯ +ç» ± +éª Ĵ +ç» ² +éª ĵ +ç» ¶ +ç» º +ç» » +ç» ¾ +éª ĸ +ç¼ ģ +èĢ ł +çIJ « +çIJ µ +çIJ ¶ +çIJ ¥ +çIJ ¨ +çIJ ° +çIJ ® +çIJ ¯ +çIJ ¬ +çIJ ļ +è¾ ĩ +é¼ ĭ +æı ³ +åł ŀ +æIJ ½ +æı ¸ +æı ł +åł Ļ +è¶ Ħ +æı ĸ +é¢ ī +å¡ Ħ +æı ¿ +èĢ ĭ +æı Ħ +èĽ © +èĽ ° +å¡ Ĩ +æij Ĵ +æı Ĩ +æİ ¾ +èģ Ĵ +èij ij +èij ļ +éĿ ° +éĿ ¸ +èij ³ +èij º +èij ¸ +èIJ ¼ +èij ¶ +è ĴĮ +èij Ń +æ¥ ® +æ £¼ +æ¤ Ł +æ£ ¹ +æ¤ ¤ +æ£ ° +èµ į +æ¤ ĭ +æ¤ ģ +æ¤ ª +æ¤ IJ +é¹ ģ +éħ ¤ +éħ ¢ +éħ ¡ +é¹ Ĥ +æ® ļ +æ® Ľ +éĽ ± +è¾ ĭ +æ¤ ł +è¾ İ +çĿ Ħ +çĿ ĩ +çĿ ĥ +æĪ ¢ +åĸ ĭ +åĹ Ĵ +åĸ ĥ +åĸ ± +åĸ ¹ +æĻ · +åĸ Ī +è· ĸ +è· Ĺ +è· ŀ +è· ļ +è· İ +è· ı +è· Ĩ +èĽ ± +èĽ ² +èĽ Ń +èĽ ³ +èĽ IJ +èĽ Ķ +èĽ ŀ +èĽ ´ +èĽ ĺ +åĸ ģ +åĸ Ł +åķ ¾ +åĹ ĸ +åĸ ij +åĹ Ł +åĹ ŀ +åĸ Ļ +åµ ĺ +åµ ĸ +å´ ´ +éģ Ħ +è© Ī +åµ İ +å µ¬ +åµ Ľ +åµ ¯ +åµ Ŀ +åµ « +å¹ Ħ +åµ ĭ +èµ ķ +éĵ » +éĵ ¼ +éĵ ¿ +éĶ ĥ +éĶ Ĩ +éĶ ĩ +éĶ ī +éĶ ı +éĶ ij +éĶ Ĵ +éĶ Ķ +éĶ ķ +æİ £ +çŁ ¬ +æ° ° +æ¯ ³ +æ¯ ½ +çĬ Ĭ +çĬ Ħ +çĬ ĭ +é ¹Ħ +çĬ į +åµ ĩ +é» į +ç¨ ĥ +ç¨ Ĥ +çŃ ļ +çŃ µ +çŃ Į +åĤ £ +åĤ Ī +èĪ Ħ +çī į +åĤ ¥ +åĤ § +éģ ij +åĤ © +å¾ ¨ +åª Ń +çķ ² +å¼ ij +ç¿ ķ +é¹ Ĩ +èħ Ī +èħ ĵ +èħ Ĩ +èħ ´ +èħ ļ +èħ ± +é± ¿ +é² Ģ +é² Ĥ +çĮ ¢ +çĮ ¹ +çĮ ¥ +é£ ĵ +è§ ŀ +è§ ļ +çĮ ± +é¢ İ +é£ § +é¦ ĩ +é¦ Ĭ +äº µ +èĦ Ķ +è£ Ĵ +çĹ £ +çĹ ¨ +çĹ ¦ +çĹ ŀ +çĹ ¤ +çĹ § +èµ ĵ +ç« ¦ +çĵ ¿ +åķ » +é¢ ı +é¹ ĩ +éĺ ij +éĺ Ĵ +éĺ ķ +ç² ŀ +éģ Ĵ +åŃ ³ +çĦ ¯ +çĦ ľ +çĦ ± +é¹ Ī +æ¸ « +æ¹ ® +æ¹ İ +æ¹ ľ +æ¹ į +æ¹ « +æº ² +æ¹ Ł +æº Ĩ +æ¹ ² +æ¹ Ķ +æ¹ ī +æ¸ ¥ +æ» ģ +æĦ ł +æĥ º +æĦ ¦ +æĥ ´ +æĦ Ģ +æĦ İ +æĦ Ķ +åĸ ¾ +å¯ IJ +è° Ł +è£ ¢ +è£ İ +è£ ¥ +ç¥ ¾ +è° ł +è° ¡ +è° ¥ +è° § +åŃ ± +å¼ ¼ +å· ½ +éª ĺ +åª ª +å· ¯ +ç¿ ļ +çļ ´ +éª Ľ +ç¼ Ĥ +ç¼ ĥ +ç¼ Ħ +å½ ĺ +ç¼ ĩ +ç¼ Ī +ç¼ Į +ç¼ ij +ç¼ Ĵ +ç¼ Ĺ +é£ ¨ +èĢ ¢ +çij ģ +çij Ĺ +çij Ħ +éģ ¨ +éª ľ +éŁ « +é« ¡ +å¡ ¬ +éĦ ¢ +è¶ Ķ +è¶ ij +æij ħ +æij ģ +èľ ĩ +æIJ ĭ +æIJ ª +æIJ IJ +æIJ Ľ +æIJ ł +æij Ī +å½ Ģ +æ¯ Ĥ +æIJ ¦ +æIJ ¡ +èĵ ģ +æĪ ¡ +è ĵį +éĦ ŀ +èĵ IJ +èĵ ¦ +é¹ ĭ +èĴ ½ +èĵ ĸ +èĵ Ĭ +èĴ ¯ +èĵ Ł +èĵ ij +èĴ º +èĵ ł +èĴ Ł +èĴ ¡ +èĴ ¹ +èĴ ´ +èĴ Ĺ +èĵ ¥ +æ¥ Ķ +æ¥ Ĥ +æ¥ Ŀ +æ¥ « +æ¥ ¸ +æ¤ ´ +æ§ Į +æ¥ ¯ +çļ Ļ +æ¦ Ī +æ§ İ +æ¦ ī +æ¥ ¦ +æ¥ £ +æ¥ ¹ +æ¤ ½ +åī ½ +éħ © +èľ ĥ +ç¢ Ľ +ç¢ ĵ +ç¡ ¼ +ç¢ ī +ç¢ ļ +ç¢ ĩ +ç¢ ľ +é¹ Į +è¾ ı +é¾ ĥ +é¾ ħ +è¨ ¾ +ç² ² +çĿ ļ +åĹ ª +éŁ ª +åĹ · +åĹ ī +çĿ ¨ +çĿ ¢ +éĽ İ +çĿ ¥ +åĹ ij +åĹ « +åĹ ¬ +åĹ Ķ +åĹ Ŀ +æĪ ¥ +åĹ Ħ +çħ ¦ +æļ Ħ +éģ ¢ +æ ļĮ +è· ¬ +è· ¶ +è ·¸ +è· IJ +è· £ +è· ¹ +èĽ ¸ +èľ Ĭ +èľ į +èľ ī +èľ £ +çķ ¹ +èĽ ¹ +åĹ ¥ +åĹ ² +åĹ ³ +åĹ Į +åĹ į +åĹ IJ +åĹ ¤ +åĹ µ +ç½ ¨ +åµ Ĭ +åµ ´ +éª ° +éĶ Ĺ +éĶ Ľ +éĶ ľ +éĶ Ŀ +éĶ ŀ +éĶ Ł +éĶ ¢ +éĶ ¨ +éĶ © +éĶ Ń +éĶ ± +éĽ ī +æ° ² +çĬ ı +æŃ ĥ +ç¨ ŀ +ç¨ Ĺ +ç¨ Ķ +çŃ ł +çŃ ¢ +çŃ ® +çŃ ² +çī Ĵ +æķ « +å¾ Ń +æĦ Ĩ +èī Ħ +è§ İ +æ¯ ¹ +è² Ĭ +è² ħ +è² ī +é¢ Ķ +èħ ł +èħ © +èħ ¼ +èħ Ń +è ħ§ +å¡ į +åª µ +é² ħ +é² Ĩ +é² ĩ +é² Ī +é² ĭ +é² IJ +èĤ Ħ +é¹ IJ +é£ ķ +è§ ¥ +éģ Ľ +é¦ IJ +é¹ ij +äº ¶ +çĺ ĥ +çĹ ± +çĹ ¼ +çĹ ¿ +çĺ IJ +çĺ ģ +çĺ Ĩ +éº Ĥ +æŃ Ĩ +æĹ Ĵ +éĺ ĸ +éĺ Ĺ +ç¾ § +è± ¢ +ç² ³ +çĮ · +çħ ³ +çħ ¨ +çħ ħ +çħ Ĭ +çħ ¸ +çħ º +æ» Ł +æº ± +æº ĺ +æ¼ Ń +æ» ¢ +æº ¥ +æº ½ +è£ Ł +æº » +æº · +æ» Ĺ +æ» « +æº ´ +æ» ı +æ» ĥ +æ» ¦ +æº ı +æ» Ĥ +æ» ĵ +æº Ł +æ» ª +æĦ « +æħ Ĭ +é² İ +éª ŀ +çª ł +çª £ +è£ ± +è£ ¨ +è£ ¾ +è£ ° +ç¦ Ĭ +è° © +è° ª +åª ¾ +å« « +åª ² +å« Ĵ +å« Ķ +åª ¸ +ç¼ Ļ +ç¼ ľ +ç¼ Ľ +è¾ Ķ +éª Ŀ +ç¼ Ł +ç¼ ¡ +ç¼ ¢ +ç¼ £ +éª Ł +èĢ ¥ +çĴ Ī +çij Ń +çį Ĵ +è§ ı +æħ Ŀ +å« ł +åı Ĩ +æij ½ +å¢ ģ +æĴ Ĥ +æij ŀ +æĴ Ħ +ç¿ ¥ +è¸ ħ +æij Ń +å¢ ī +å¢ Ĵ +æ¦ ĸ +ç¶ ¦ +èĶ « +èĶ · +éĿ º +éĿ ¼ +éŀ ħ +éĿ ¿ +çĶ į +èĶ ¸ +èĶ Ł +èĶ º +æĪ ¬ +èķ ĸ +èĶ » +èĵ ¿ +æĸ ¡ +é¹ ķ +èĵ ¼ +æ¦ Ľ +æ¦ § +æ¦ « +æ¦ Ń +æ§ Ķ +æ¦ ± +æ§ ģ +æ§ ł +æ¦ · +åĥ ° +éħ ½ +éħ ¹ +ç¢ ¡ +ç¢ ´ +ç¢ £ +ç¢ ² +èĩ § +è± ¨ +æ® ¡ +éľ ģ +èľ ļ +é¾ ĩ +é¾ Ī +ä ģ +äģ ĸ +çĿ ½ +åĺ ŀ +åĺ Ī +åĺ Į +åĺ ģ +æļ Ŀ +è¸ Į +è¸ ī +èľ ŀ +èľ ¥ +èľ ® +èĿ Ī +èľ ´ +èľ ± +èľ © +èľ · +èľ ¿ +èŀ Ĥ +èľ ¢ +åĺ ¡ +é¹ Ĺ +åĺ £ +åĺ ¤ +åĺ ļ +åĹ ¾ +åĺ § +ç½ ´ +ç½ ± +å¹ Ķ +å¶ Ĥ +å¹ Ľ +èµ Ļ +ç½ Ĥ +éª · +éª ¶ +é¹ ĺ +éĶ ² +éĶ ´ +éĶ ¶ +éĶ · +éĶ ¸ +éĶ µ +éķ Ĥ +çĬ Ĵ +ç® IJ +ç® ¦ +ç® § +ç® ¸ +ç® ¬ +ç® ħ +ç® ª +ç® ľ +ç® ¢ +ç® ĵ +åĥ ĸ +åĦ Ĩ +åĥ ³ +åĥ Ń +åĬ ģ +åĥ ® +éŃ ĥ +éŃ Ĩ +çĿ ¾ +èī ĭ +éĦ ± +èĨ Ī +èĨ ij +é² ij +é² Ķ +é² ļ +é² Ľ +é² Ł +çį IJ +è§ « +éĽ Ĵ +å¤ ¤ +é¦ ij +éĬ ® +å¡ ¾ +çĺ Į +çĺ Ĭ +çĺ ĺ +çĺ Ļ +æĹ ĸ +èĨ Ĥ +éĺ ļ +éĦ ¯ +é² ŀ +ç² ¿ +ç² ¼ +ç³ ģ +æ§ Ĭ +é¹ ļ +çĨ ĺ +çĨ ¥ +æ½ ¢ +æ¼ ķ +æ» ¹ +æ¼ ¯ +æ¼ ¶ +æ½ ĭ +æ½ ´ +æ¼ ª +æ¼ ī +æ¼ © +æ¾ ī +æħ µ +æIJ ´ +çª ¨ +å¯ ¤ +ç¶ ® +è° ® +è¤ ¡ +è¤ Ļ +è¤ ĵ +è¤ Ľ +è¤ Ĭ +è° ¯ +è° ° +è° ² +å± £ +é¹ Ľ +å« ± +å« ĸ +å« ¦ +å« ļ +å «ĺ +é¼ IJ +çŀ Ģ +é¹ ľ +éª ł +ç¼ ¥ +ç¼ ¦ +ç¼ § +ç¼ ¨ +éª ¢ +ç¼ « +èĢ ¦ +èĢ § +çĴ ľ +çĴ İ +çĴ ģ +å¥ Ń +é« ¯ +é« « +æĴ · +æĴ ħ +èµ Ń +æĴ ¸ +éĭ Ĩ +æĴ Ļ +æĴ º +å¢ Ģ +èģ © +è§ IJ +éŀ ij +èķ Ļ +éŀ Ĵ +èķ Ī +èķ ¨ +èķ ¤ +èķ ŀ +èķ º +çŀ ¢ +èķ ĥ +èķ ² +èµ ľ +æ§ ¿ +æ¨ ¯ +æ§ Ń +æ¨ Ĺ +æ¨ ĺ +æ§ ² +éĨ Į +éĨ ħ +éĿ ¥ +éŃ ĩ +é¤ į +ç£ Ķ +ç£ Ļ +éľ Ī +è¾ ĺ +é¾ ī +é¾ Ĭ +è§ ij +çŀ Į +ç ŀĭ +çŀ ij +åĺ Ń +åĻ İ +åĻ ¶ +é¢ Ļ +æļ ¹ +åĻ ĺ +è¸ Ķ +è¸ Ŀ +è¸ Ł +è¸ Ĵ +è¸ ¬ +è¸ ® +è¸ ¯ +è¸ º +è¸ ŀ +èĿ ½ +èĿ ¾ +èĿ » +èĿ ° +èĿ ® +è ŀĭ +èĿ ĵ +èĿ £ +è Ŀ¼ +åĺ ¬ +é¢ ļ +åĻ į +åĻ Ļ +åĻ Į +åĻ Ķ +é¢ Ľ +å¹ ŀ +å¹ ¡ +å¶ Ļ +å¶ Ŀ +éª º +éķ Ĭ +éķ ī +éķ Į +éķ ı +éķ Ĵ +éķ ĵ +éķ Ķ +ç¨ · +ç® ´ +ç¯ ij +ç¯ ģ +ç¯ Į +çī ĸ +åĦ ĭ +èĻ ¢ +é¹ ŀ +èĨ ĺ +é² ł +é² ¡ +é² ¢ +é² £ +é² ¥ +é² § +é² © +çį Ĺ +çį ł +è§ ¯ +é¦ ĵ +é¦ Ķ +éº ¾ +å» Ľ +çĺ Ľ +çĺ ¼ +çĺ ¢ +çĺ ł +é½ ij +ç¾ ° +𥠻 +ð¥» Ĺ +ç³ Į +ç³ į +ç³ ħ +çĨ ľ +ç Ĩµ +æ¾ į +æ¾ Į +æ½ ¸ +æ½ ¦ +æ½ ² +éĭ Ī +æ½ Ł +æ½ º +å¯ ® +çª ³ +è° ³ +è¤ ´ +è¤ Ł +è¤ « +è° µ +çĨ ¨ +å± ¦ +åĭ ° +æĪ ® +èĿ ¥ +ç¼ ¬ +ç¼ ® +ç¼ ¯ +éª £ +çķ ¿ +èĢ © +èĢ ¨ +èĢ ª +çĴ Ł +éĿ Ľ +çĴ ł +çĴ ĺ +èģ ± +èŀ ¯ +é« » +é« Ń +é« ¹ +æĵ Ģ +çĶ ı +æĵ ŀ +ç¸ ł +ç£ ¬ +é¢ ŀ +èķ » +é¢ Ł +èĸ ¤ +èĸ ¨ +æª ł +èĸ ı +èĸ ® +èĸ ľ +èĸ ħ +æ¨ ¾ +æ© Ľ +æ© ĩ +æ¨ µ +æª İ +æ© ¹ +æ¨ ½ +æ¨ ¨ +æ© ¼ +å¢ ¼ +æ© IJ +ç¿ ® +éĨ IJ +éĨ į +éĨ ļ +ç£ ² +èµ Ŀ +æ® ª +éľ ı +éĮ ¾ +è¾ ļ +éģ ½ +æ° ħ +çŀ Ł +çŀ ł +çŀ ° +åļ Ħ +åļ Ĩ +åĻ ¤ +æļ ¾ +è¹ Ģ +è¸ µ +è¸ ½ +è¹ ī +è¹ ģ +èŀ ¨ +èŀ Ī +èŀ ħ +èŀ Ń +èŀ ł +èŀ Ł +åĻ ± +åĻ « +åĻ » +åĻ ¼ +ç½ ¹ +åľ ľ +ä ¦ +ä¦ ĥ +éķ Ĺ +éķ ĺ +éķ ļ +éķ Ľ +éķ Ŀ +éķ ŀ +éķ ł +æ° ĩ +æ° Ĩ +ç© ij +ç¯ Ŀ +ç¯ ¥ +ç¯ ¦ +ç¯ ª +ç¯ Ļ +çĽ ¥ +åĬ ĵ +ç¿ ± +éŃ ī +éŃ Ī +å¾ ¼ +æŃ Ļ +èĨ ¦ +èĨ Ļ +é² ® +é² ± +é² ³ +é² ´ +é² µ +é² · +é² » +çį ´ +çį Ń +çį ¬ +éĤ Ĥ +é¹ § +å» ¨ +èµ Ł +çĺ ° +å» ª +çĺ ¿ +çĺ µ +çĺ ´ +çĻ ĥ +çĺ ³ +éº ĩ +éº Ī +å ¬´ +å£ ħ +ç³ Ĺ +çĶ ij +çĩ İ +çĩ ł +çĩ Ķ +çĩ § +æ¿ ij +æ¿ ī +æ½ ŀ +æ¾ § +æ¾ ¹ +æ¾ ¥ +æ¾ ¶ +æ¿ Ĥ +è¤ ° +çª ¸ +å¬ ĸ +çĬ Ł +éļ ° +å¬ Ĺ +é¢ ¡ +ç¼ ± +ç¼ ² +ç¼ ³ +çĴ © +çĴ ª +èŀ « +æĵ ¤ +å£ ķ +è§ ³ +ç½ Ħ +æĵ ¢ +èĸ ¹ +éŀ ¡ +éŀ ¬ +èĸ · +èĹ ĵ +èĹ ģ +æª Ħ +æª © +æĩ ĭ +éĨ ¢ +ç¿ ³ +ç¤ ħ +ç£ ´ +é¹ © +é¾ ĭ +é¾ Į +è± ³ +å£ ij +é» » +åļ ı +åļ ħ +è¹ ij +è¹ Ĵ +è¹ Ĭ +è Ł¥ +èŀ ¬ +èŀ µ +çĸ ĥ +èŀ ³ +èŁ ij +åļ ĵ +ç½ ½ +ç½ ¾ +å¶ · +é» ľ +é» Ŀ +é« ģ +é« Ģ +éķ ¡ +éķ ¢ +éķ £ +éķ ¦ +éķ § +éķ © +éķ ª +éķ « +ç½ ħ +ç° Į +ç¯ ¾ +ç¯ ¼ +ç° ĸ +ç° ĭ +é¼ ¢ +åĦ ¡ +é¹ ª +é¼ ¾ +çļ ¤ +éŃ į +é¾ ł +ç¹ ĩ +è² ĺ +éĤ Ī +è² Ķ +èĩ Į +èĨ » +èĩ Ĩ +èĩ ĥ +é² ¼ +é² ½ +é³ Ģ +é³ ĥ +é³ ħ +é³ ĩ +é³ Ĭ +èŀ ½ +çĩ ® +é¹ « +ç³ ľ +ç¸ » +çĻ į +éº ĭ +æĩ ij +æ¿ ¡ +æ¿ ® +æ¿ ŀ +æ¿ ł +æ¿ ¯ +è¹ ĩ +è¬ ĩ +éĤ ĥ +è¥ ģ +æª Ĺ +æ ĵĺ +åŃ º +éļ ³ +å¬ · +èŁ Ĭ +é¹ ¬ +éį ª +éı Ĭ +é¬ Ī +é¬ ĥ +çŀ ½ +éŀ ¯ +éŀ ¨ +éŀ « +éŀ § +éŀ £ +èĹ ľ +èĹ ł +éĨ ª +è¹ Ļ +ç¤ ĵ +çĩ ¹ +é¤ ® +çŀ ¿ +æĽ Ľ +é¢ ¢ +èº ĩ +è¹ ļ +èŁ Ľ +èŁ ª +èŁ ł +èŁ ® +é¹ ® +é» ł +é» Ł +é« ħ +é« Ĥ +éķ ¬ +éķ Ń +éķ ¯ +é¦ ¥ +ç° Ł +ç° ª +é¼ ¬ +éĽ ł +èī Ł +é³ İ +é³ ı +é³ IJ +çĻ ŀ +çĻ Ķ +ç³ ¨ +è¹ © +éİ ı +éĤ ĭ +é¬ ı +æĶ ī +éŀ ² +éŀ ´ +èĹ ¿ +èĺ § +èĺ ħ +éĨ ® +éĨ ¯ +éħ ĥ +éľ ª +éľ Ń +éľ ¨ +é» ¼ +åļ ¯ +è¹ ° +è¹ ¶ +è¹ ½ +è¹ ¼ +è¹ ´ +è¹ ¾ +è¹ ¿ +èł ĸ +èł ĵ +èŁ ¾ +èł Ĭ +é» ¢ +é« ĭ +é« Į +éķ ² +ç± Ģ +é½ ģ +éŃ ij +èī ¨ +é³ ĵ +é³ Ķ +é³ ķ +é³ Ĺ +é³ Ļ +éı ĸ +ç¾ ¸ +㸠Ĩ +çĢ £ +çĢ Ľ +è¥ ¦ +è° ¶ +è¥ ŀ +éª ¥ +ç¼ µ +çĵ Ĵ +æĶ ĺ +èĺ © +èĺ ĸ +éĨ ´ +éľ ° +éħ Ĩ +çŁ į +èº ħ +é¼ į +å· ī +é» © +é» ¥ +é» ª +éķ ³ +éķ ´ +é» § +çº Ĥ +çĴ º +é¼ ¯ +èĩ ľ +é³ ľ +é³ Ŀ +é³ Ł +çį ¾ +åŃ Ģ +éª § +ç ĵĺ +é¼ Ļ +éĨ º +ç¤ ´ +é¢ ¦ +æĽ © +é³ ¢ +éº Ŀ +å¤ Ķ +çĪ Ŀ +çģ ı +ç¦ ³ +éIJ ¾ +ç¾ ¼ +èł ¡ +èĢ ± +é¹ ³ +æ° į +é¥ ķ +èº IJ +é« ij +éķ µ +ç© ° +é¥ Ķ +é¬ » +é¬ Ł +è¶ ± +æĶ « +æĶ ¥ +é¢ § +èº ľ +é¼ ¹ +çĻ ¯ +èł ² +èł ¹ +èº ŀ +è¡ ¢ +çģ ŀ +è¥ » +çº Ľ +é¬ £ +æĶ ® +åĽ Ķ +é¦ ķ +æĪ Ĩ +çĪ ¨ +é½ ī +äº į +å° ¢ +å½ ³ +åį ¬ +æ® ³ +ðł Ļ¶ +æ¯ Į +éĤ ĺ +æĪ ĭ +åľ ¢ +æ° ķ +ä¼ ĭ +ä» Ŀ +åĨ ® +æ° ¿ +æ± Ī +æ° ¾ +å¿ ī +å® Ħ +𬣠Ļ +è® ± +æī ŀ +åľ ² +åľ « +èĬ ı +èĬ ĥ +æľ ³ +æľ ¸ +ð¨ Ļ +ð¨Ļ ¸ +éĤ ¨ +åIJ Ĵ +åIJ ĸ +å± ¼ +å± ¾ +è¾ ¿ +éĴ Ĩ +ä» ³ +ä¼ £ +ä¼ Ī +çĻ ¿ +çĶ ª +éĤ ł +çĬ ´ +åĨ ± +éĤ ¡ +ð¬ĩ ķ +æ± ĭ +ä ľ +äľ £ +è® » +𬣠ŀ +åŃ ĸ +ð¬ĺ ĵ +çº © +çİ Ĵ +çİ ĵ +çİ ĺ +çİ ļ +åĪ ¬ +ð«Ń Ł +åĿ ľ +åĿ ī +æī ½ +ð«Ń ¢ +åĿ ĭ +æī º +㧠ij +æ¯ IJ +èĬ ° +èĬ £ +èĭ Ĭ +èĭ ī +èĬ ĺ +èĬ ´ +èĬ ł +ð« ĩ +ð«ĩ Ń +èĬ ¤ +æĿ ķ +æĿ Ļ +æĿ Ħ +æĿ § +æĿ © +å° ª +å° ¨ +è½ ª +ð«IJ Ħ +åĿ Ĵ +èĬ Ī +æĹ ´ +æĹ µ +åij Ļ +ã ķ +ãķ ® +å² į +ð« µ +𫵠· +å² ł +å² ľ +åij ĩ +åĨ ı +è§ ĥ +å² Ļ +ä¼ ¾ +ãij ĩ +ä¼ Ń +ä½ ĸ +ä¼ ² +ä½ ģ +é£ ı +çĭ ĥ +éĹ ¶ +æ± § +æ± « +𣲠ĺ +ð£² Ĺ +æ² Ħ +æ² ĺ +ð¬ĩ Ļ +æ± Ń +ã³ ĩ +æ² ĩ +å¿ ® +å¿ ³ +å¿ º +𬣠¡ +ç¥ ĥ +è¯ ĩ +éĤ ² +è¯ İ +è¯ IJ +å± ĥ +ð« ¸ +𫸠© +å² Ĭ +éĺ ½ +ä¢ º +éĺ ¼ +å¦ § +å¦ ĺ +ð¨ ļ +ð¨ļ ķ +çº ® +é© ² +ð«ĺ ľ +çº » +ð¬ĺ ĺ +ð«ĺ Ŀ +çº ¼ +çİ ¤ +çİ ŀ +çİ ± +çİ Ł +éĤ ½ +éĤ ¿ +åĿ ¥ +åĿ ° +åĿ ¬ +åĿ ½ +å¼ Ĩ +èĢ µ +ä¢ ¼ +ð¦ Ń +ð¦Ń ľ +èĮ ĭ +èĭ § +èĭ ¾ +èĭ ł +æŀ ħ +ãŃ İ +æŀ ĺ +æŀ į +çŁ ¼ +çŁ » +åĮ ¼ +𬨠Ĥ +ð¬Ģ © +ð¬Ģ ª +æĹ ¿ +æĺ Ħ +æĺ Ĵ +æĺ Ī +åĴ ī +åĴ ĩ +åĴ į +å² µ +å² ½ +å² ¨ +å² ŀ +å³ Ĥ +ã Ł +ãŁ ĥ +åĽ · +𬬠© +éĴ IJ +éĴ Ķ +éĴ ĸ +çī ¥ +ä½ ´ +åŀ Ī +ä¾ ģ +ä¾ ¹ +ä½ ¸ +ä½ º +éļ ¹ +ãij Ĭ +ä¾ Ĥ +ä½ ½ +ä¾ ĺ +éĥ Ī +èĪ ł +éĥ IJ +éĥ ĥ +æĶ ½ +èĤ Ń +èĤ ¸ +èĤ · +çĭ ī +çĭ Ŀ +é¥ ³ +å¿ ŀ +çĤ Į +çĤ Ĩ +æ³ Ļ +æ² º +æ³ Ĥ +æ³ ľ +æ³ ĥ +æ³ ĩ +æĢ Ĭ +å³ ĥ +ç© ¸ +ç¥ ĭ +ç¥ Ĭ +ð«į £ +𬣠³ +𬠩½ +é¸ ¤ +å¼ ¢ +å¼ ¨ +éĻ ij +𬮠¿ +éĻ İ +𬯠Ģ +åį º +ä¹ ¸ +å¦ Ń +å§ Ī +ð« ° +ð«° Ľ +è¿ ³ +åı ķ +𬳠µ +é© µ +𬳠¶ +ä Į +äĮ ¹ +é© º +ð«ł Ĭ +ç» ĭ +ç» IJ +çł ī +èĢ Ķ +ãĽ ĥ +çİ ¶ +çı ĩ +çı ħ +ð¬į Ľ +çı ĭ +çİ ¹ +çı Į +çİ ¿ +éŁ ¨ +åŀ ļ +åŀ ¯ +åŀ Ļ +åŀ ² +åŁ ı +åŀ į +èĢ ĩ +é¿ į +åŀ İ +åŀ ´ +åŀ Ł +åŀ ŀ +æĮ ĵ +åŀ µ +åŀ ı +æĭ ¶ +èį ĸ +èį ģ +èį Ļ +èį Ľ +èĮ Ī +èĮ ½ +èį Ħ +èĮ º +ð¬ľ ¬ +èį ĵ +èĮ ³ +𦠰 +𦰠¡ +èĮ Ľ +èį Ń +ãŃ ķ +æŁ · +æŁ ĥ +æŁ Ĭ +æŀ ¹ +æł IJ +æŁ ĸ +éĥ ļ +åī ħ +ä´ ĵ +è¿ º +åİ ĸ +çł Ĩ +çł ij +çł Ħ +èĢ ı +å¥ ĵ +ä ¶ +ä¶ ® +è½ µ +è½ · +è½ ¹ +è½ º +æĺ º +𪠾 +𪾠¢ +æĺ ½ +çĽ · +åĴ ¡ +åĴ º +æĺ ³ +æĺ £ +æĺ ¤ +æĺ « +æĺ ¡ +åĴ ¥ +æĺ ª +èĻ · +èĻ ¸ +åĵ ĥ +å³ ĺ +èĢ ij +å³ Ľ +𪨠° +å³ Ĺ +å³ § +å¸ ¡ +éĴ ĺ +ð«ĵ § +éĴ ľ +𬬠® +𬬠± +ð¬¬ Ń +éĴ ª +éĴ ¬ +éĴ Ń +çŁ § +ç§ ¬ +ä¿ « +èĪ ģ +ä¿ ľ +ä¿ Ļ +ä¿ į +åŀ ķ +è¡ İ +èĪ £ +å¼ ĩ +ä¾ ´ +é¸ § +äı ¡ +èĥ ł +ð¦ Ļ¶ +èĥ Ī +èĥ © +èĥ £ +æľ ı +é£ IJ +è¨ Ħ +é¥ » +åº ¤ +çĸ ¢ +çĤ £ +çĤ Ł +ã ¶ +㶠² +æ´ Ń +æ´ ĺ +æ´ ĵ +æ´ ¿ +ã³ ļ +æ³ ļ +æµ Ī +æµ ī +æ´ ¸ +æ´ ij +æ´ ¢ +æ´ Ī +æ´ ļ +æ´ º +æ´ ¨ +æµ IJ +ã³ ĺ +æ´ ´ +æ´ £ +æģ Ķ +å® ¬ +çª Ģ +æī Ĥ +è¢ Ĩ +ç¥ ı +ç¥ IJ +ç¥ ķ +åı ļ +éĻ § +éĻ ŀ +å¨ Ģ +å§ ŀ +å§ ± +å§ ¤ +å§ ¶ +å§ ½ +æŀ ² +ç» ĸ +éª ĥ +ð¬ĺ ¡ +𬳠½ +ð¬ĺ © +ð«Ħ § +å½ ĸ +éª ī +æģ Ŀ +çı ª +çı Ľ +çı ¹ +çIJ Ĭ +çİ ¼ +çı ĸ +ðª Ł +ðªŁ Ŀ +çı ½ +çı ¦ +çı « +çı Ĵ +ð¬į ¤ +çı ¢ +çı ķ +çı Ŀ +ð«Ń ¼ +åŁ Ĺ +åŀ ¾ +åŀ º +åŁ Ĩ +åŀ ¿ +åŁ Į +åŁ ĩ +èİ ° +èĮ Ŀ +ð¬ľ ¯ +éĦ Ģ +èİ ¶ +èİ Ŀ +äĵ ĸ +èİ Ļ +æł » +æ¡ ł +ð¬ Ĥ +ð¬Ĥ © +æ¡ Ħ +æ¢ ł +æł ´ +æ¢ ´ +æł Ĵ +éħ İ +éħ ı +ð«ł Ĩ +çł µ +çł ł +çł « +çł ¬ +ç¡ ģ +æģ § +ç¿ ĥ +éĥ ª +ð¨ IJ +ð¨IJ Ī +è¾ Ģ +è¾ ģ +ð¬ Į +ð¬Į Ĺ +åī ķ +èµ Ģ +åĵ ¢ +æĻ ħ +æĻ Ĭ +åĶ Ŀ +åĵ ³ +åĵ ± +åĨ Ķ +æĻ Ķ +æĻ IJ +çķ ĸ +èļ Ħ +èļ Ĩ +ð« ij +ð«ij ¡ +å¸ ± +å´ ģ +å³ ¿ +𪨠¶ +å´ Ħ +å¸ ¨ +å ´Ģ +èµ Ĩ +𬠬¸ +éĴ · +𬬠» +𬬠¹ +𬬠¿ +ð¬Ń ģ +çľ ļ +çĶ ¡ +ç¬ « +åĢ » +åĢ ´ +èĦ © +åĢ ® +åĢ ķ +åĢ ŀ +ð« ¢ +ð«¢ ¸ +åĢ ĵ +åĢ § +è¡ ĥ +èĻ Ĵ +èĪ Ń +èĪ ¯ +èĪ ¥ +çĵ ŀ +é¬ ¯ +é¸ ° +èĦ İ +æľ ĵ +èĥ ² +èĻ ĵ +é± ½ +çĭ ´ +å³ ± +çĭ » +çľ ¢ +ð«Ĺ § +åĭ į +çĹ Ħ +çĸ ° +çĹ ĥ +ç« ĺ +ç¾ ĸ +ç¾ ĵ +æ¡ Ĭ +æķ ī +çĥ ł +çĥ Ķ +çĥ ¶ +çĥ » +ð¬Ĭ Ī +æ¶ į +æµ ¡ +æµ Ń +æµ ¬ +æ¶ Ħ +æ¶ ¢ +æ¶ IJ +æµ ° +æµ Ł +æµ Ľ +æµ ¼ +æµ ² +æ¶ ĺ +æĤ Ī +æĤ ĥ +æĤ ¢ +ð¬Ĵ Ī +å® § +çª ħ +çª Ĭ +çª İ +æī ħ +æī Ĩ +è¢ ª +è¢ Ĺ +è¢ ¯ +ç¥ § +éļ º +åł ² +çĸ į +𨠺 +𨺠Ļ +éĻ ´ +ç ĥĿ +çł ® +ãĽ ļ +åĵ ¿ +ç¿ Ģ +ç¿ Ĥ +åī Ł +𬳠¿ +ð«Ħ ¨ +ç» ¤ +éª į +ð¬ĺ « +ä Ĥ +äĤ ® +çIJ İ +çı ¸ +çı µ +çIJ Ħ +çIJ Ī +çIJ Ģ +çı º +æİ Ń +åł İ +åł IJ +åŁ ¼ +æİ İ +åŁ « +åł Į +æĻ ¢ +ð« ® +ð«® ĥ +æİ ŀ +åŁ ª +å£ ¸ +ãĻ į +èģ į +èı Ŀ +èIJ ļ +èı ¥ +èİ ¿ +äĵ « +åĭ ļ +äĵ ¬ +èIJ Ĩ +èı Ĥ +èı į +èı ¼ +èIJ £ +äĵ ¨ +èı ī +äĵ Ľ +æ¢ ¼ +æ¢ ½ +æ¡ ² +æ¢ ¾ +æ¡ ¯ +æ¢ £ +æ¢ Į +æ¡ ¹ +æķ Ķ +åİ £ +ç¡ Ķ +é¿ İ +ç¡ Ļ +ç¡ ļ +ç¡ Ĭ +ç¡ į +åĭ Ķ +ä´ ķ +é¾ ģ +éĢ ´ +åĶ ª +åķ « +ç¿ Ī +ã « +ã« ° +æĻ Ļ +çķ ¤ +𬱠ĸ +è¶ ¼ +è· Ĥ +èĽ ĥ +èļ ² +ð¬Ł ½ +èļ º +åķ ´ +äİ ĥ +å´ § +å´ Ł +å´ ŀ +å´ Ĵ +å´ Į +å´ ¡ +éĵ ı +ð«ĵ ¯ +ð«Ł ¹ +éĵ ķ +ð«Ł ¼ +éĵ ĸ +éĵ ĺ +éĵ ļ +éĵ ŀ +éĵ ¥ +éĵ ´ +çī » +çī ¿ +ç¨ Ĩ +ç¬ ± +ç¬ ¯ +åģ ° +åģ ¡ +é¸ º +åģ Ń +åģ ² +åģ ģ +ã ¿ +ã¿ ł +éĦ ħ +åģ ĵ +å¾ Ľ +è¡ Ĵ +èĪ ³ +èĪ ² +é¸ ¼ +æĤ Ĩ +éĦ ĥ +çĵ » +ä Ŀ +äĿ Ļ +èĦ ¶ +èĦ ŀ +èĦ Ł +äı ² +é± ¾ +çĮ ĩ +çĮ Ĭ +çĮ Ħ +è§ ĸ +ðł ħ +ðłħ ¤ +åº ± +åº ¼ +åº ³ +çĹ ĵ +ä´ Ķ +ç« « +åł ĥ +éĺ Į +ç¾ Ŀ +ç¾ ķ +çĦ Ĩ +çĥ º +çĦ Į +æ· ı +ð¬ĩ ¹ +æ· Ł +æ· ľ +æ· ´ +æ· ¯ +æ¹ ´ +æ¶ ´ +ð¬į ¡ +ã ¥ +㥠Ħ +æĥ Ľ +æĥ Ķ +æĤ ° +æĥ Ļ +å¯ ģ +éĢ Ń +𬤠ĩ +ð«į ¯ +è¢ ¼ +è£ Ī +ç¥ ² +𬤠Ĭ +ð«į ² +è° ŀ +èī ´ +å¼ ¸ +å¼ ¶ +ð¬¯ İ +éļ ĥ +å© ŀ +å¨ µ +å© ¼ +åª ĸ +å© ³ +å© į +å© Į +å© « +å© ¤ +å© ĺ +å© ł +ð¬ĺ ¬ +ð¬ĺ Ń +𬴠Ĥ +ð«ĺ ¦ +ç» ¹ +ð«Ł ħ +ð¬ĺ ¯ +éª ķ +ð«ĺ § +çµ ľ +çı · +çIJ ² +çIJ ¡ +çIJ Ł +çIJ Ķ +çIJ Ń +åł ¾ +åł ¼ +æı ķ +ãĻ ĺ +åł § +åĸ Ĩ +åł ¨ +å¡ ħ +åł ł +çµ · +𪠣 +𪣠» +ð¡ İ +ð¡İ ļ +è ijľ +æĥ İ +èIJ ³ +èij Ļ +éĿ ¬ +èij ´ +èĴ ĩ +èĴ Ī +éĦ ļ +èĴ ī +èĵ ĩ +èIJ © +èij ° +èij İ +éĦ ij +èĴ İ +èij ĸ +èĴ Ħ +èIJ ¹ +æ£ ¤ +æ£ ½ +æ£ « +æ¤ ĵ +æ¤ ij +ð¬ ĥ +ð¬ĥ Ĭ +é¹ Ģ +æ¤ Ĩ +æ£ ĵ +æ£ ¬ +æ£ ª +æ¤ Ģ +æ¥ Ĺ +𬠷 +𬷠ķ +çĶ ¦ +éħ ¦ +è§ Į +å¥ ¡ +çļ ķ +ç¡ ª +æ¬ ¹ +è© Ł +ð«IJ IJ +è¾ Į +æ£ IJ +é¾ Ĥ +𬠹 +𬹠¼ +é» ¹ +çī ļ +çĿ İ +æĻ « +æĻ ª +æĻ ± +ð § +𧠿 +𧿠¹ +èĽ ij +çķ ¯ +æĸ Ŀ +åĸ ¤ +å´ ¶ +åµ ģ +ð« ¶ +𫶠ĩ +å´ ¾ +åµ ħ +å´ ¿ +åµ ļ +ç¿ Ļ +ð«ĸ ® +åľ Į +åľ IJ +èµ ij +èµ Ĵ +é¿ ı +éĵ ¹ +ð¬Ń Ĭ +éĵ ½ +𨱠ĩ +ð«ĵ ¶ +éĶ Ĭ +éĶ į +éĶ İ +ð¬Ń İ +éĶ ĵ +çĬ ĩ +é¢ ĭ +ç¨ Į +çŃ Ģ +çŃ ĺ +çŃ ľ +çŃ ¥ +çŃ ħ +åĤ ĥ +åĤ ī +ç¿ Ľ +åĤ Ĵ +åĤ ķ +èĪ ¾ +çķ ¬ +ð«ĸ ¯ +èĦ ¿ +èħ ĺ +ä IJ +äIJ ĥ +èħ Ļ +èħ Ĵ +𬱠Ł +é² ĥ +çĮ ° +ð« Ľ +ð«Ľ Ń +çĮ ¯ +ã º +㺠Ħ +é¦ ī +åĩ ĵ +éĦ Ĺ +ð« · +ð«· · +å» ĭ +å» Ĩ +éĦ Į +ç² ¢ +éģ Ĩ +æĹ IJ +𬮠± +çĦ ŀ +ð¬Ĭ ¤ +æ¬ » +𣠸 +𣸠£ +æº ļ +æº ģ +æ¹ Ŀ +æ¸ ° +æ¹ ĵ +ã ´ +ã´ Ķ +æ¸ Ł +æº ł +æ¸ ¼ +æº ĩ +æ¹ £ +æ¹ ij +æº ŀ +æĦ IJ +æĦ ĥ +æķ © +çĶ ¯ +æ£ ¨ +æī Ĭ +è£ £ +ç¥ ¼ +å© » +åª Ĩ +åª ŀ +ãĽ ¹ +åª ĵ +åª Ĥ +åª Ħ +æ¯ µ +çŁ ŀ +𬴠ĥ +ð«ĺ ¨ +ç¼ Ĭ +ç¼ IJ +éª Ļ +çij ĥ +çij ĵ +çij ħ +çij Ĩ +ä´ ĸ +çij ĸ +çij Ŀ +çij Ķ +çij Ģ +𤠧 +𤧠Ľ +çij ³ +çij Ĥ +å¶ ħ +çij ij +éģ ĺ +é« ¢ +å¡ ¥ +åł ½ +èµ ª +æij Ľ +å¡ Ŀ +æIJ Ĵ +æIJ Į +èĴ ± +èĴ ¨ +èĵ ı +èĶ Ģ +èĵ ¢ +èĵ Ĥ +èĴ » +èĵ £ +æ¤ ¹ +æ¥ ª +æ¦ ĥ +æ¦ ħ +æ¥ Ĵ +æ¥ © +æ¦ ĩ +æ¤ ¸ +æ¥ Ļ +æŃ ħ +𬠪 +𬪠© +ç¢ ĥ +ç¢ ı +ð¬Ĵ Ķ +ç¢ Ī +äĥ ħ +ç¡ ¿ +éĦ ł +è¾ Ĵ +ð¬¨ İ +ð«IJ ĵ +é¾ Ĩ +è§ ľ +ä £ +ä£ ĺ +æļ ķ +é¹ į +ð« « +ð«« ĩ +㬠Ĭ +æļ ħ +è· ± +èľ IJ +èľ İ +åµ ² +èµ Ĺ +éª ± +éĶ ĸ +ð«ĵ ¹ +éĶ ĺ +éĶ ³ +éĶ § +éĶ ª +ð¬Ń ļ +éĶ « +éĶ ¬ +ð¬Ń Ľ +ç¨ ij +ç¨ Ļ +ä ħ +äħ Ł +ð¬ ķ +ð¬ķ Ĥ +çŃ » +çŃ ¼ +çŃ ¶ +çŃ ¦ +çŃ ¤ +åĤ º +é¹ İ +åĥ ĩ +èī ħ +èī ī +è° ¼ +è² Ĩ +èħ ½ +èħ ¨ +èħ ¯ +é² ī +é² Ĭ +é² Į +ä² Ł +𬶠ĭ +𬶠į +é² ı +éĽ Ĭ +çĮ º +é£ Ķ +è§ Ł +ð¦ Ŀ¼ +é¦ Į +è£ Ľ +å» Ĵ +çĺ ħ +éĦ ĺ +é¹ Ĵ +éĦ ľ +éº Ģ +éĦ £ +éĺ ĺ +ð«Ķ ¶ +çħ ģ +çħ ĥ +çħ ´ +çħ ĭ +çħ Ł +çħ ĵ +æ» ł +æº į +æº ¹ +æ» Ĩ +æ» ī +æº ¦ +æº µ +æ¼ · +æ» § +æ» ĺ +æ» į +æĦ Ń +æħ ¥ +æħ Ĩ +å¡ ± +ð« ĮĢ +è £¼ +ç¦ ĭ +ç¦ Ķ +ç¦ ĺ +ç¦ Ĵ +è° « +é¹ Ķ +ð«ĸ ³ +æĦ į +å« Ħ +åª ± +æĪ ¤ +åĭ ł +æĪ £ +ð«ĺ ª +ð«ĺ ¬ +ç¼ ŀ +èĢ ¤ +çij § +ð« ŀ +ð«ŀ © +çij ¨ +çij ± +çij · +çij ¢ +æĸ ł +æij ı +å¢ ķ +å¢ Ī +å¢ IJ +å¢ ĺ +æij ´ +éĬ İ +ð¡ IJ +ð¡IJ ĵ +å¢ ļ +æĴ ĸ +𪠤 +ðª¤ Ĺ +éĿ ½ +éŀ ģ +èĶ Į +èĶ Ī +èĵ ° +èĶ ¹ +èĶ Ĭ +åĺ ı +æ¦ ° +æ¦ ij +æ§ ļ +ð£ Ĺ +ð£Ĺ ĭ +æ§ ľ +æ¦ į +çĸ IJ +𬸠ĺ +éħ º +éħ ¾ +éħ ² +éħ ´ +ç¢ ¶ +äĥ İ +ð¬Ĵ Ĺ +ç¢ ¨ +ð¥ Ķ +ð¥Ķ ² +ç¢ ¹ +ç¢ ¥ +åĬ Ĥ +ð«ļ ĸ +ä´ Ĺ +å¤ ¥ +çŀ į +é¹ ĸ +ã¬ İ +è· ½ +èľ ¾ +å¹ ĸ +å¶ į +åľ Ļ +𨱠ı +éĶ º +éĶ ¼ +éĶ ½ +ð¬Ń ¤ +éĶ ¾ +éĶ ¿ +éķ ĥ +éķ Ħ +éķ ħ +é¦ Ŀ +é¹ Ļ +ç® ¨ +ç® ĸ +åĬ Ħ +åĥ ¬ +åĥ ¦ +åĥ Ķ +åĥ İ +æ§ ĥ +ãĻ ¦ +é² Ĵ +é² ķ +ð«ļ ķ +é² ĸ +é² Ĺ +é² ĺ +é² Ļ +𬶠IJ +𬶠ı +ð ©½ +𩽠¾ +å¤ IJ +çį į +é£ Ĺ +𬸠ļ +åĩ ĺ +å» ij +å» Ļ +çĺ Ĺ +çĺ ¥ +çĺ ķ +é² Ŀ +éĦ « +çĨ ĩ +æ¼ ¹ +æ¼ ĸ +æ½ Ĩ +æ¼ ¤ +æ½ © +æ¼ ¼ +æ¼ ´ +ã ½ +ã½ ı +æ¼ Ī +æ¼ ĭ +æ¼ » +æħ ¬ +çª ¬ +çª Ń +ã ® +ã® ¾ +𬤠Ŀ +è¤ ķ +ç¦ Ľ +ç¦ ļ +éļ © +å« ķ +å« Ń +å« ľ +å« ª +ð¬ ĻĤ +ã » +ã» ¬ +éº ¹ +çĴ Ĩ +æ¼ ¦ +åı ĩ +å¢ £ +å¢ ¦ +å¢ ¡ +åĬ IJ +èĸ ģ +èķ ° +èĶ ĥ +é¼ Ĵ +æ§ ± +é¹ Ŀ +ç£ ı +ç£ ī +æ® £ +æħ Ń +éľ ħ +æļ µ +æļ ² +æļ ¶ +è¸ ¦ +è¸ £ +äĹ ĸ +èĿ ĺ +èĿ ² +èĿ ¤ +åĻ ĩ +å ĻĤ +åĻ Ģ +ç½ ¶ +å¶ ² +å¶ ĵ +ãł ĩ +å¶ Ł +å¶ Ĵ +éķ Ĩ +éķ Ī +éķ ĭ +éķ İ +ð¬Ń © +éķ ķ +ç¨ ¹ +åĦ ĩ +çļ ŀ +çļ Ľ +ä´ ĺ +èī İ +èī ı +é¹ Ł +𩾠ĥ +é² ¦ +é² ª +é² ¬ +æ© ¥ +è§ Ń +é¹ ł +é¹ ¡ +ç³ ĩ +ç³ Ī +ç¿ ¦ +é¹ ¢ +é¹ £ +çĨ Ľ +æ½ ĸ +æ½ µ +ã µ +ãµ IJ +æ¾ Ĥ +æ¾ Ľ +çij ¬ +æ½ ½ +æ½ ¾ +æ½ ı +æĨ Ń +æĨ ķ +𬸠£ +æĪ Ń +è¤ ¯ +ç¦ ¤ +ð«į ½ +å« ½ +éģ ¹ +𬴠Ĭ +çĴ ¥ +çĴ ² +çĴ Ĵ +æĨ Ļ +æĵ IJ +éĦ ¹ +èĸ ³ +éŀ Ķ +é» ĩ +ð¬ ŀ +ð¬ŀ Ł +èķ Ĺ +èĸ ¢ +èķ ¹ +æ© ŀ +æ© ij +æ© ¦ +éĨ ij +è§ ± +ç£ ¡ +ð¥ ķ +ð¥ķ ¢ +ç£ ľ +è± ® +ð«Ł ¦ +𬺠Ī +ð«ł ľ +é¹ ¾ +èĻ ¤ +æļ ¿ +æĽ Į +æĽ Ī +㬠ļ +è¹ ħ +è¸ ¶ +äĹ Ľ +èŀ Ĺ +çĸ ģ +ãł ĵ +å¹ ª +𪠩 +𪩠ĺ +å¶ ¦ +ð¬Ń ¬ +𨱠ij +ð¬Ń ¯ +é¦ ŀ +ç© Ħ +ç¯ ļ +ç¯ ¯ +ç° ī +é¼ ½ +è¡ ł +çĽ ¦ +èŀ £ +ç¸ ¢ +é² Ń +é² ¯ +é² ° +é² º +é² ¹ +ð«Ĺ ´ +äº ¸ +çĻ Ģ +çĺ Ń +𬸠¦ +ç¾ ± +ç³ Ĵ +çĩ ĭ +çĨ » +çĩ Ĭ +çĩ ļ +çĩ ı +æ¿ © +æ¿ ĭ +æ¾ ª +æ¾ ½ +æ¾ ´ +æ¾ Ń +æ¾ ¼ +æĨ · +æĨ º +æĩ Ķ +é» ī +å¬ Ľ +é¹ ¨ +ç¿ ¯ +ð«Ħ · +çĴ ± +𤠩½ +çĴ ¬ +çĴ ® +é« ½ +æĵ ¿ +èĸ ¿ +èĸ ¸ +æª ij +æ« Ĩ +æª ŀ +éĨ ¨ +ç ¹Ħ +ç£ ¹ +ç£ » +çŀ « +çŀ µ +è¹ IJ +èŁ ı +ã ĺ +ãĺ İ +ð¬Ń ³ +éķ ¤ +ð¬Ń ¶ +ð«Ķ į +éķ ¥ +éķ ¨ +ð¬Ń ¸ +𨱠Ķ +ð¬Ń ¼ +ð«Ķ İ +çŁ ° +ç© Ļ +ç© ľ +ç© Ł +ç° ķ +ç° ĥ +ç° ı +åĦ ¦ +éŃ ĭ +æĸ ¶ +èī ļ +𬸠ª +è° ¿ +ä² ł +𬶠Ł +é² ¾ +𬶠ł +é² ¿ +é³ ģ +é³ Ĥ +é³ Ī +é³ ī +çį ¯ +äĹ ª +é¦ ĺ +è¥ ķ +è¥ ļ +𬶠¨ +èŀ ± +çĶ ĵ +å¬ ¬ +å¬ ¥ +ð¦ Ī +ð¦Ī ¡ +ð«Ħ ¸ +çĵ Ģ +éĩ IJ +é¬ ¶ +çĪ ĩ +éŀ ³ +éŀ ® +ð¬Ł ģ +èĹ Ł +èĹ ¦ +èĹ ¨ +é¹ ² +æª « +é» ¡ +ç¤ ŀ +ç¤ Į +ð¥ ĸ +ð¥ĸ ¨ +è¹ ¢ +è¹ ľ +èŁ « +äĹ ´ +åļ ļ +é« ĥ +éķ ® +éķ ± +éħ Ĥ +é¦ § +ç° ł +ç° Ŀ +ç° ° +é¼ « +é¼ © +çļ ¦ +èĩ ij +ä² ¢ +é³ ij +é³ Ĵ +é¹ ± +é¹ ¯ +çĻ Ĺ +ð¦ Ĵ +ð¦Ĵ į +æĹ ŀ +ç¿ · +åĨ ģ +äİ ĸ +çĢ Ķ +çĢ į +çĢ Į +è¥ ľ +ä´ Ļ +ð¬Ļ Ĭ +åļ Ń +ã ° +ã° Ģ +é¬ · +éĨ Ń +è¹ ¯ +èł ĭ +ç¿ ¾ +é³ ĺ +åĦ ³ +åĦ ´ +é¼ Ĺ +ð¬¶ Ń +𩾠Į +é³ ļ +é³ Ľ +éº ij +éº ĸ +èł ĥ +å½ Ł +å¬ ¿ +é¬ Ĵ +èĺ ĺ +æ¬ Ĥ +é Ĩµ +é¢ ¥ +çĶ Ĺ +ð¨ Ł +ð¨Ł ł +å· ĩ +éħ ħ +é« İ +çĬ ¨ +𬶠® +ð¨ Ń +ð¨Ń ī +㸠Į +çĪ Ķ +çĢ ± +çĢ ¹ +çĢ ¼ +çĢ µ +è¥ « +åŃ ħ +éª ¦ +ð¬Ļ ĭ +èĢ ° +𤠫 +𤫠ī +çĵ ĸ +é¬ ĺ +è¶ ¯ +𬺠ĵ +ç½ į +é¼ ± +é³ ł +é³ ¡ +é³ £ +çĪ Ł +çĪ ļ +çģ Ī +éŁ Ĥ +ç³ µ +èĺ ¼ +ç¤ µ +é¹ ´ +èº Ķ +çļ Ń +é¾ ¢ +é³ ¤ +äº ¹ +ç± ¥ +é¼ · +ð«ļ Ń +çİ ĥ +éĨ ¾ +é½ ĩ +è§ ¿ +èł ¼ +× § +× ¤ +× Ľ +×ķ× ª +× ¡ +×Ļ× Ŀ +× ¦ +× Ĵ +× ĺ +×ķ× ¨ +× Ŀ +×ķ× ľ +× ĸ +๠Ĥ +ï º +ðŁ į +ðŁ IJ +×Ļ× ¨ +ï » +ðŁ ij +ðĿ IJ +ðŁ ı +ðŁ Ķ +ðŁ Į +ðŁ İ +ðŁ ĵ +× Ł +ðĿ ij +×ķ× ĵ +ï ¦ +Ġ× ķ +×ķ× ij +à¸Ń à¸ĩ +ðĿ ĺ +×Ļ× ª +ðĿ ķ +à¸Ĺ ีà¹Ī +Ø§Ø ¦ +ðŁ ¤ +×ķ× Ł +ر ÙĬ +×Ļ× ľ +ร ะ +า ย +ï ¯ +ï ® +า ม +â ĩ +ðŁ ¥ +ï Ń +ðĿ Ļ +×ķ× ł +á ½ +Ġ× Ľ +ðŁ ļ +â ļ +ï § +×ij ר +×Ļ× ł +á ´ +Ġ× Ĺ +á ¼ +ðĿ Ĺ +Ġ× ¢ +×Ļ× Ķ +ãģ£ ãģŁ +ãģĵ ãģ¨ +á ¸ +ÙĬ ÙĨ +ãģª ãģĦ +ا ع +ภ¨ +à¹Ī à¸ĩ +×Ļ× ĵ +×ŀ ש +á Ī +׳ ×Ļ +×Ļ× ij +ï ¥ +ðĿ ĵ +Ġ× Ļ +× ļ +ั à¸ĩ +â ĵ +ï ¤ +ĠاÙĦ Ø£ +า à¸ģ +à¹ī à¸Ļ +à¹Ģ ร +×ķ× Ŀ +á ¹ +ภ¶ +×Ļ× § +ภĭ +à¸Ħ ร +ภĺ +ั à¸ģ +ðŁ ķ +ÙĪ ÙĨ +à¸Ń ย +â Ĭ +ðĿ Ĵ +ĠاÙĦ ع +า à¸Ļ +×Ļ× Ł +ÙĦ ÙĬ +×Ļ× © +à¸Ľ ระ +à¹Ģ à¸Ľ +Ġ× ł +×ķ× ¡ +ภł +Ùħ ÙĨ +×ķ× ¢ +×ķ× ŀ +â Į +ðŁ § +à¹ĩ à¸Ļ +ภį +ã İ +á µ +ĠاÙĦ س +×ķ× § +ห ล +ðŁ ĩ +â ı +ðŁ ¦ +Ġ×Ķ ×ŀ +ÙĪ ا +Ġ× ª +ר ×IJ +à¸Ń à¸Ļ +ภ© +à¹Ī ว +×ķ× ¦ +í Ĺ +ã Ħ +ï ¨ +ï ¹ +â İ +ï ² +ðĿ ļ +ð IJ +à¸Ħ ว +ห à¸Ļ +Ġ× ¨ +ب ÙĬ +ร à¹Į +ر ا +Ø´ ر +×ķ× Ĺ +×ķ× ¤ +×ķ× © +×ķ× Ĵ +í Ŀ +â Ľ +à¸ķ ิ +à¹Ģ à¸ģ +ï ³ +ï ± +à¸Ķ à¹ī +ë ¹ +ï ¬ +á ¿ +ðŁ Ľ +ðĿ ĸ +à¹Īา à¸ĩ +ู à¹ī +Ġ×Ķ ×IJ +ĠاÙĦ ØŃ +פ ר +ÙĪ Ùħ +à¹Ģ ล +í ĸ +×Ļ× ¢ +ì Ī +í ĵ +ðŁ ħ +á ł +à¸Ħว าม +à¸Ī ะ +׳ ×Ķ +Ġ× § +ภŁ +à¹ī à¸ĩ +ห ม +ت Ùħ +׾ ×Ļ +ÙĬ د +à¹Ī à¸Ļ +׊ר +ש ר +à¹Ģ à¸Ĺ +×ŀ ר +ë ĸ +ع ÙĦ +×ŀ ×¢ +â ² +׾ ×Ķ +Ġ× ¤ +à¸Ń à¸ģ +س ÙĦ +×Ļ× ŀ +ÙĤ ÙĬ +í İ +ت ØŃ +×Ļ× ¡ +×Ļ× Ĺ +í Ľ +ï ° +â ½ +á ī +á Ĭ +á ¨ +Ùĩ ا +Ġ׾ ×Ķ +×ķ× IJ +Ùħ ا +à¹īà¸Ń à¸ĩ +ر ب +ĠاÙĦ ج +×ŀ ×ĵ +Ùħ ÙĦ +ت ر +à¹Ģ à¸Ķ +ק ר +í ħ +ì ¼ +ê ¿ +ã Ī +á IJ +ðŁ Ĺ +ê ¦ +á ĭ +ðĿ Ķ +à¹Ģà¸Ľ à¹ĩà¸Ļ +à¹ĥ ห +ม า +ว à¹Īา +ม ี +ี à¹ī +à¹Ħม à¹Ī +ÙĨ ÙĬ +Ø ¤ +ร า +×ķ ×Ļ +ãĤĪ ãģĨ +ิ à¸Ķ +×Ļ× ¤ +׊׾ +ÙĤ د +à¹Ģ ส +×Ļ× ĺ +à¸ģ ล +ר ׼ +×ķ× Ľ +×Ļ× Ľ +ë Ī +ë ĥ +ðŁ ĸ +á ħ +â ¼ +ã ī +à¹Ħ à¸Ķà¹ī +ת ×Ļ +×Ļ× IJ +ĠاÙĦ Ø¥ +à¸ł า +ร ิ +ÙĤ Ø© +ØŃ Ø¯ +ê » +ì ± +ת ×Ĺ +ì º +â ĭ +á Ħ +á ¾ +â µ +â ¾ +ĠÙĪ اÙĦ +׳ ×ķ +Ù Ģ +ÙĬ ا +à¸ģ à¹ĩ +×ŀ ×Ķ +ãģĦ ãĤĭ +ع د +ĠاÙĦ ÙĨ +Ġ×Ķ ש +Ø ¦ +ั à¹īà¸ĩ +ร ัà¸ļ +ÙĪ ÙĤ +ãģ§ ãģį +à¹Ģ à¸ŀ +׼ ׾ +×ĺ ר +ั à¸Ķ +à¸Ń า +ì ¢ +à¸Ń à¸ļ +à¸ķ ร +à¹Ģ à¸Ĭ +ì Ķ +ãģĹ ãģ¾ +ë ģ +ë ķ +ðŁ Ļ +â Ĵ +á ¶ +à¹ģ ล +ÙĨ ا +à¹ĥห à¹ī +à¹Ħ à¸Ľ +× £ +ั ว +า à¸ĩ +×ĵ ר +×ij ׾ +פ ×Ļ +Ġ× ĵ +ĠاÙĦ Ùģ +à¹Ģ à¸Ĥ +ש ×Ķ +×IJ ר +ë ¬ +ãģ« ãģª +ÑĢ о +ว ิ +Ùħ ر +×IJ ת +Ùĥ ر +س ب +ÙĨ ت +ãģĹ ãģĦ +ا ج +à¸Ń รà¹Į +Ùĥ ÙĦ +س Ùħ +ส ิ +×Ļ× ¦ +ë Ŀ +í ľ +ì ī +á Ĩ +Ùĩ Ùħ +à¸Ļ ีà¹ī +ãģĤ ãĤĭ +ãģĦ ãģ¦ +س ÙĬ +׾ ×IJ +د ر +ãģ ļ +ÙĪ ج +ĠاÙĦ Ø® +ص ر +í ı +à¹īา à¸ĩ +ุ à¸Ķ +×ķ× ĺ +×ij ×¢ +í Ĩ +à¸Ĭ า +ร ม +ש ×ŀ +×ŀ ס +ê ´ +ì ´ +ë ľ +ì ¿ +ì © +ë » +â ¤ +ðŁ Ĩ +á Į +á ķ +Ø° ا +à¸Ĺ ำ +à¸ķ à¹Ī +ĠاÙĦ ÙĤ +ÙĦ Ùĥ +ู à¹Ī +à¸Ħ ุ +ÙĬ Ùħ +׳ ×Ļ×Ŀ +ืà¹Ī à¸Ń +ÙĪ ع +ãĤ ĩ +ا ÙĤ +Ġ×ij ×¢ +à¹Ģ ม +ج Ùħ +á» « +ãģĵãģ¨ ãģĮ +ب د +×ķ× Ķ +ש ׾ +Ùĩ ر +à¹Ģ à¸Ļ +ãģ ¹ +í ĭ +ì » +ì ½ +ë Ń +ì Į +í Ģ +ë Į +ë º +ã Ĭ +à¹ĥ à¸Ļ +Ġ× Ĵ +๠Ĩ +à¸Ī าà¸ģ +ว ย +à¹ĥ à¸Ĭ +à¸ĩ าà¸Ļ +ĠاÙĦ Ø´ +ا ØŃ +à¹īา à¸Ļ +ืà¹Ī à¸Ńà¸ĩ +×IJ ×Ļ +ب ÙĦ +ãģ¨ æĢĿ +׳ ס +ãģ¾ ãģĽ +Ùĥ ÙĨ +×¢ ר +ĠاÙĦ د +ש ת +í ŀ +Ùħ س +ص ÙĦ +×ķ׳ ×Ķ +ار Ø© +ÙĦ Ùħ +ส ม +Ø£ ÙĨ +ת ר +×IJ ×ŀ +ع ب +Ø® ت +ãĤ ĥ +ì ¡ +ì £ +ив а +ส ั +ึ à¸ģ +ì ¸ +ë Ĩ +алÑĮ н +ì ³ +ì į +ê ¼ +ê ½ +ì ı +ã Į +ã ı +ï © +ê ª +á İ +Ġ× ĸ +à¸ģ ัà¸Ļ +×Ļ ×ķ +à¸Ħ à¸Ļ +׳ ×ķת +à¸ľ ูà¹ī +à¹ĥ à¸Ī +ãģĦ ãģŁ +Ùģ ر +×ĺ ×Ļ +צ ×Ļ +ãĤĤ ãģ® +ĠاÙĦ ص +ãģ¾ãģĽ ãĤĵ +د Ø© +×ij ×Ļ +ĠاÙĦ ر +Ġ×ŀ ×IJ +ส ำ +à¹Ģ ห +ع ر +ãģª ãģı +à¸ģร ะ +×ij ×ĵ +à¹Ģ à¸Ī +×Ļ× ļ +×Ĺ ×Ļ +ÙĬ ع +ש ×ij +ÙĨ Ø© +ÙĪ ض +ÙĦ Ùģ +ÙĢ ÙĢ +פ ×¢ +í Ī +×ŀ ק +ภIJ +ØŃ Ø© +ا ص +Ñĭв а +à¸Ħ ม +ว ั +à¸Ľ ล +ì Ł +í ļ +ë ´ +ë ij +ë ī +ë ĩ +ì ¨ +ë ± +ë İ +â ¬ +á ¥ +á Ĺ +á Ľ +á į +Å © +à¸Ķ ี +ô i +Ġ× ¡ +׾ ×ķ +á»Ŀ i +à¸Ħุ à¸ĵ +â y +à¸Ļ า +×Ĺ ×ĵ +×ĵ ×Ļ +ห า +ج ÙĦ +à¹Ģ ว +ãĤĩ ãģĨ +Ùħ Ø© +ĠاÙĦ Ùĥ +Ġ×Ķ ×¢ +ج ر +×ĸ ר +ا Ø· +׼ ת +×ķ׳ ×Ļ×Ŀ +ØŃ Ùħ +ê ¶ +ر Ùĥ +Ġ׾ ×¢ +×ķ× ĸ +ส ร +צ ׾ +Ø ¢ +ا ست +à¹Ī ม +Ø® ر +צ ×¢ +×Ļר ×ķת +اد Ø© +Ø´ ار +×ŀ ×Ĺ +í Ĵ +à¹Ģร ีย +׊ק +Ø§Ø « +ร à¸ĩ +à¹Ģ à¸ķ +à¸Ī ำ +ภĿ +à¹Īา ย +à¸Ħ ล +ÙĤ ÙĪ +иÑĩеÑģ к +à¸ĵ à¹Į +ั ย +Ùħ ع +ë ¨ +ë ¿ +ë ® +ï ´ +ì ¥ +ì « +ë µ +á ¡ +â į +ð ĵ +â ° +à¸Ĥ à¸Ńà¸ĩ +Ù ĭ +à¸ģ ัà¸ļ +ãģ® ãģ§ +à¹ī ว +à¸Ńย à¹Īาà¸ĩ +ãģ Ń +á»ĩ t +à¸ķ à¹īà¸Ńà¸ĩ +×ŀ ×Ļ +à¹ģ à¸ļ +×Ĵ ר +ÙĪ Ùģ +ÙĤ ÙĦ +à¸łà¸² à¸ŀ +ר ×Ļ +ล า +ÙĬ س +Ġ× ¦ +ÙĬ Ùģ +Ġ× ĺ +à¸ľ ล +á ng +ร ว +Ġ×ŀ ש +×IJ ×ķת +×ĸ ×Ķ +ู à¸ģ +à¸Ļ ัà¸ģ +اÙĨ ÙĬ +د ا +ãģ ³ +׼ ף +ãĤī ãĤĮ +ãĤĮ ãģ° +ת ק +ú c +ÙĪ ز +×Ļר ×Ķ +Ġn gh +án h +Ġ×ķ ×IJ +á» ħ +ส ุà¸Ķ +ë į° +ا ض +اÙĦ ÙĬ +ب ار +ع Ùħ +à¸ļ า +ت ج +à¸ŀ ร +×ķר ×Ķ +ả ng +Ø® ÙĦ +ภī +ắ c +ש ×Ļ×Ŀ +í Ķ +Ùģ س +×Ļ× Ĵ +п ÑĢ +ĠاÙĦ Ø« +س Ø· +ร ูà¹ī +ีà¹Ī ย +à¸Ń à¸Ķ +ãģª ãĤĬ +×Ĵ ×ĵ +ãģĦ ãģ¾ãģĹãģŁ +ס ק +Ø® ص +la ÅŁ +ен но +ب ØŃ +ส à¸Ļ +ภ® +ר×IJ ש +Ùħ ÙĪ +دÙĬ د +ษ า +×ķ× ļ +ãĥ§ ãĥ³ +à¸ķ ุ +Ġê µ +ĠÑģв о +צ ×ij +à¸Ń ม +à¸Ľ ร +ت ع +×Ķ ת +اÙħ ÙĦ +×ŀ ׳ +ç ¶ļ +ภ¤ +í į +ë ĺ +ë ¤ +ì ij +â ´ +ã ĭ +Ġب اÙĦ +á»ģ u +ĠاÙĦ ÙĦ +à¸ķ ัว +Ø° Ùĩ +ึ à¸ĩ +à¹ĥà¸Ĭ à¹ī +á»ĵ ng +à¸Ļ ั +ม าà¸ģ +ãĥ Ł +×ŀ ×ķ +à¸Ĺ ย +á»Ļ i +Ạ± +ả o +à¹Ĥ à¸Ķ +×IJ ׾ +ส าม +ÙĪ ب +à¸Ĺ ุ +ย ัà¸ĩ +×¢ ת +×ķ׳ ×ķת +à¸Ĥ ึ +à¸Ĥึ à¹īà¸Ļ +à¸ģ à¹Ī +Ạ« +á»ij c +ãģĹ ãĤĩãģĨ +á»ĭ ch +Ġ×IJ ×ķת +Ġש ×IJ +׼ ×ķ׾ +á»Ļ c +ع Ø© +à¸Ĺ ี +à¹Ģ à¸Ń +Ùĥ ت +ãģ » +Ạ» +ìĹ ħ +à¸Ń à¸Ńà¸ģ +اÙĨ ت +à¹Ħ ร +Ġ×IJ ×Ĺר +Ø· ر +ÙĨ د +ื à¹īà¸Ń +Ø· ÙĦ +×IJ ×Ķ +uy ên +í ĸī +×ij ×Ķ +à¸Ħ à¹Ī +à¸Ĭ à¹Īว +ãģĤãĤĬ ãģ¾ãģĻ +ÙĬ ب +ק ׾ +ãĥ Ļ +Ä © +س ر +า ว +ãĤ ± +à¸ļ ริ +ר ×Ĵ +á»ĥ u +ØŃ Øª +×ķ×ŀ ×Ļ +ب ÙĨ +êµ IJ +ÄŁ u +ãģª ãĤĵ +×ij ק +Ġפ ר +ắ n +ØŃ ÙĦ +×ij ×Ĺ +ấ u +×ij ×ķ×ĵ +ãĥ ¯ +Ġ׾ ק +ั à¸į +à¸ŀ ิ +×Ĺ ×Ķ +×ĸ ׼ +ãĥ¼ãĥ ł +ÑĤ елÑĮ +×ŀ ×Ļ×ĵ +ÙĬ Ø® +Ạ³ +ت ص +à¸ĺ ิ +è¾ ¼ +ì ĵ +Ùĥ Ø© +ÙĤ ب +à¸Ħ à¹Į +à¹īา ย +à¸ĵ ะ +า ะ +ë Ĵ +ê ¾ +ë · +ì ĩ +ê º +ì ģ +ë Ģ +ì ¾ +ë ½ +ë ļ +ì Ń +ì İ +á ij +ë Ĺ +ê Ĵ +à ¡ +à ¬ +ðIJ Į +ã ĩ +ðĿ Ħ +Ġ׾ ×IJ +ãģ¨ ãģĦãģĨ +Ġn hi +×Ļ ×ķת +Ġש ×Ķ +à¹ģล à¹īว +Æ°á»Ľ c +à¸Ķà¹ī วย +à¸Ĺ าà¸ĩ +׳ ת +פ ת +à¹ģ à¸ķà¹Ī +Æ° ng +à¸Ńย ูà¹Ī +à¹ī ำ +Ġ×IJ ׾ +Ùĥ Ùħ +ấ p +ล à¸ĩ +ãģŁ ãĤģ +×Ĵ ׾ +ห ร +ĠÑĢ е +à¹Ģà¸Ĥ à¹īา +ÙĤ ر +Ġ×Ķ ס +ÙĪ ÙĬ +สาม าร +สามาร à¸ĸ +Äĥ n +à¸Ń ี +פ ×ķ +×Ļ׳ ×ķ +ว ัà¸Ļ +ặ c +íķ Ļ +×ŀ ת +ê u +Ạ¹ +Ùģ ÙĬ +×ŀ צ +à¸Ħ า +ãģĿ ãģĨ +ãĢ ħ +ا ز +ا Ùĩ +ר ×Ļ×Ŀ +ấ n +ห าร +ạ t +ÙĨ Ùĩ +à¹Ģ à¸Ħร +ج Ùĩ +׼ ×Ļ +ắ t +à¸Ħ à¹īา +ر Ø© +ãĥ ı +Ùĥ ÙĪÙĨ +ứ ng +Ġìļ ° +ย à¹Į +à¹Īว à¸Ļ +à¸ģ ำ +Ø« ر +Ñģ и +ĠاÙĦ Ø· +Ġ×Ķ צ +ĠØ · +ĠاÙĦ ÙĪ +ê¹ Į +ØŃ ÙĬ +ار ات +à¹Ģ à¸ĭ +ب ا +г ÑĢ +ร ี +ืà¸Ń à¸Ļ +ع ت +ÙĤ اÙĦ +د Ùħ +Ø ¡ +Ġ×ŀ ק +×ĵ ×Ļ×Ŀ +×¢ ׾ +ãģ Ĵ +ëĭ ĺ +×¢ ×Ķ +Ġìĸ ´ +Ñģ ÑĮ +ÙĤ Ø· +ãĥ Ľ +èĢĥ ãģĪ +à¹ģ à¸Ļ +ÙĪ ات +â u +ĠìĤ¬ ëŀ +ห ว +ĠاÙĦØ£ Ùħ +Ġ×Ķ ×ŀש +ب ÙĪ +à¸Ĭ à¸Ļ +ãĤĵ ãģ§ãģĻ +ว à¸Ļ +à¸ģร รม +×ŀ ×ķ×ĵ +Ùĥ اÙĨ +×ķ× £ +ол ог +ت ÙĨ +à¸ķ à¹Į +ê² ĥ +ר ×ĺ +ừ ng +×ķ×ij ×Ķ +Ùħ ØŃ +ĠÐ § +פ ×Ĵ +ส à¸ĸ +ãģĭ ãĤĬ +ını z +à¹Ģ ย +ãĥ¼ ãĥ³ +ãģĬ ãĤĬ +פ ש +ิ à¸ķ +Ø· ÙĨ +×Ļת ×Ļ +×IJ ׳ +ç ek +ì ª +×ŀ ×ij +ศ า +ãĤ¹ ãĤ¿ +à¸ļ ุ +×ĵ ×ijר +ãģĦ ãģı +ส ะ +à¹Ģ หล +ิ à¸ĩ +à¸ŀ ัà¸Ļ +ãģĦ ãģŁãģł +ãĤĤ ãĤī +à¹ī ม +ãģĵãģ¨ãģĮ ãģ§ãģį +าร à¹Į +ุ à¸ĩ +í ij +ì ¯ +ë ¼ +í Ĥ +ì · +ê ¡ +á ı +á Ĵ +ðĿ ľ +á © +ðŁ Ħ +ðIJ ¤ +Ġש ׾ +Ġ×ŀ ×Ķ +à¹ģล ะ +Ġ׼ ׾ +Ạ½ +á»Ļ ng +Ø° ÙĬ +л е +× ¥ +ãģª ãģ© +ĠÙĪ Ø£ +หà¸Ļ à¹īา +ãģ¾ ãģ§ +à¸ķà¹Ī à¸Ń +à¸Ĺ ัà¹īà¸ĩ +ãģł ãģij +à¹ģà¸ļ à¸ļ +à¹Ģร า +פ ׾ +ãģŁ ãģĦ +à¹Ģล ย +ãģ£ãģ¦ ãģĦãĤĭ +ế p +ึ à¹Īà¸ĩ +ê ´Ģ +ê³ Ħ +׼ ×ķ +à¹Ģร ืà¹Īà¸Ńà¸ĩ +ק ×Ļ +êµ Ń +פ ס +ت ÙĬ +ãĥ Ħ +Ġ×Ķ ×Ĺ +г и +ר×IJ ׾ +×ŀ ׾ +ĠØ£ ÙĬ +Ġع ÙĦÙĬ +ãģĭ ãģ£ãģŁ +ש ×Ļ +д Ñĥ +×ŀ ף +׳ ×ĺ +׳ ×Ļת +mi ÅŁ +׼ ×Ŀ +Ġ×ij ר +Ġ׾ ×ij +ĠÐ Ľ +ç e +×ķ׳ ×Ļ +ãĤĪãģĨ ãģ« +פ ×ķר +ãĥ į +Ùĥ ÙĬ +×Ĺ ×ª +Ùģ ÙĦ +Ġ×Ķ ק +Ġ×Ķ ×ij +Ġ×ŀ ס +à¹Īา à¸Ļ +п еÑĢ +à¹Īา ว +Ġ×ij ×IJ +ĠÙĪ Ùĩ +à¸Ļ ำ +Ġ×ij ש +׳ ק +ãģ© ãģĨ +ש ×ķת +×ĵ ×Ķ +à¹Ģ à¸ļ +ÙĨ س +Ġìļ° 리 +ส à¹Īวà¸Ļ +ล ัà¸ĩ +ج ز +Ġ×Ĺ ×Ļ +Ùĥ ثر +ล ะ +Ùĩ د +ĠÙĪ ب +اÙĦ Ùħ +à¹ģ ม +Æ¡ i +Ġ×ij ×Ĺ +ữ a +à¹Ģà¸Ĺ ศ +à¸ķ ัà¹īà¸ĩ +ог да +׾ ק +د د +สร à¹īาà¸ĩ +à¸Ĭ ี +Ùģ ض +à¹ģ ห +uy á»ĩn +ร ัà¸ģ +á»ĩ m +ส า +פ ק +ีย à¸ĩ +à¸ķ à¹Īาà¸ĩ +à¸Ħร ัà¹īà¸ĩ +ØŃ ÙĤ +à¹Ģ à¸Ńà¸ĩ +ائ ÙĬ +×ĺ ×¢ +اÙĦ Ø© +ิ à¹Īม +ãĤ ½ +د Ùī +Ġר ×IJ +ãģ£ ãģ¨ +ãĥĥ ãĥĹ +ÙĬر Ø© +ê± ´ +×ŀ ×IJ +×ķ ×ķ +ب ع +ãģ ² +ร าย +×ĵ ×Ŀ +ت Ùģ +à¸ķ à¸ģ +ạ ng +ãĤĴ è¦ĭ +à¸Ĭ ั +Æ°á» Ł +Æ°á»Ł ng +ج ب +×ķ×ŀ ר +ĠìĤ¬ëŀ Į +ó ng +ร ั +Ġ×Ķ ×ĸ +ר צ +Ġ×Ĺ ×ĵ +Ø° ÙĦÙĥ +×ķר ×Ļ +ãģ¡ ãĤĥ +Ùģ ع +Ġ׾ צ +á i +à¹ĩ à¸ļ +ãģ İ +à¸ģ ิ +ạ c +ë© ° +ãģª ãĤĭ +×ķ׾ ×Ŀ +à¹ģ à¸Ĺ +×ķ× ¥ +м еÑĤ +ü ÅŁ +ÑĢ Ñı +ภĴ +ÑģÑĤ оÑı +ع ÙĪد +Ùħ ار +Ø· Ø© +à¸ŀ ื +к ÑĢ +à¹ģ à¸ģ +à¹Ĥ รà¸ĩ +×ij ×Ļ×ĺ +ê² ł +×ķ׾ ×Ķ +ØŃ Ø± +ืà¹Ī à¸Ńà¸Ļ +×ķ×ij ר +׊ש +ãĥķãĤ ¡ +×ŀ ×ĺ +ú t +Ġd ön +ắ ng +ëł ĩ +ẳ ng +ว à¸ģ +ص د +Ø® Ø· +à¸Ń ั +ãĤı ãĤĮ +سÙĦ اÙħ +à¹Ģร à¹ĩ +×Ļש ×Ļ +ج اÙĦ +ãģij ãĤĭ +à¸Ĭา à¸ķิ +ÙĪا ÙĤ +à¹Ĥ à¸Ļ +ãģ¦ ãģĹãģ¾ +اع Ø© +ãĤŃ ãĥ£ +à¸į า +ÙĦا ÙĤ +ิ à¸ģ +ĠÑģ ов +ÑĢаРº +×Ļ׳ ×Ļ +ü ÄŁ +Ã¼ÄŁ ü +ק ×ij +à¹Ī à¸Ńà¸ĩ +Ġger çek +à¸Ĺ ั +ов аниÑı +×ŀ ׼ +س Ø© +×Ļ× £ +le ÅŁ +Ùħ ؤ +ĠìĿ ĺ +à¸IJ าà¸Ļ +ĠÑģ об +Ġêµ Ń +×¢ צ +з в +ส à¸ĩ +ز ÙĦ +ãģı ãĤĮ +и ÑĢÑĥ +ت Ø£ +п олн +ìĺ Ģ +ÙĨ Ø´ +׼ ×IJ +Ùħ Ø´ +à¸Ķ à¹Į +ÙĪ ÙĬÙĦ +à¹ģ à¸Ĥ +ãģ£ãģ¦ ãģĹãģ¾ +но ÑģÑĤ +в л +Ùħ ÙĤ +را ج +å¤ ī +ë Ľ +â ¸ +ì IJ +à » +á ļ +â » +ê Ļ +â § +ð Ĵ +ðĿ ĩ +Ġ×IJ ת +ĠÙĦ ÙĦ +ĠØ£ ÙĨ +Ġ×ķ ×Ķ +ãģ« ãģ¯ +Ġ×Ļ ש +ت Ùĩ +ÃŃ nh +ÙĬ ات +Ġ×ij ×ŀ +à¸Ļั à¹īà¸Ļ +à¸Ļ à¹īำ +Ãł o +à¸ķ าม +ãģ® ãģ¯ +d ır +Ġn ghi +ặ t +×ŀ ×Ļ×Ŀ +ãģ¦ ãģĦãĤĭ +Ġ×ij ת +หร ืà¸Ń +Ġس ÙĬ +ãģª ãĤī +à¹Ĥà¸Ķ ย +ı yor +à¸Ńี à¸ģ +á»ĩ nh +Ñĭ м +à¸Ĺุ à¸ģ +Ġ׾ ×Ĺ +Ġ×Ķ ר +Ġ×Ķ ×Ļ +à¸ŀ ระ +à¹Ģว ลา +ĠØ º +ẫ n +m Ä±ÅŁ +׼ ×Ķ +á»ij n +ãģ§ ãģĹãĤĩãģĨ +ãĥ ¢ +à¸Ľ ี +ס ×Ļ +ãģĵ ãĤį +Ġ׾ פ +ร à¸ĸ +ê¸ Ī +à¸ģ วà¹Īา +ë ¬´ +á»į ng +ãĤĵ ãģ§ +ãĤĪãģĨ ãģª +á»ĵ i +ãĤ ¬ +ส à¹Īà¸ĩ +×Ļ׳ ×Ķ +à¸ĸ ูà¸ģ +à¸Ī ัà¸Ķ +Ġ×Ķ ×Ĵ +ãĥ ľ +×ŀ ×ķת +ÙĪ Ùĥ +ëĭ ¨ +ĠØ « +ãģ® ãģĮ +à¹Ģห à¹ĩà¸Ļ +ع ا +à¸Ļ ิ +Å ŀ +à¸Ń ะ +ãģĪ ãĤĭ +Ø« ÙĦ +ØŃÙħ د +à¹Ģà¸ģ ิà¸Ķ +פ שר +פ ×Ķ +ม ิ +ئ ÙĬس +à¸Ĺำ à¹ĥหà¹ī +×¢ ×ĵ +ìĭ ¤ +à¸Ĭà¹Īว ย +ĠاÙĦÙħ ÙĨ +ز ÙĬ +ع ÙĬ +Ġ׼ ×IJ +ạ nh +á» ¹ +ãĤĵ ãģª +ส ู +צ ר +Æ°á»Ľ ng +×ķ ×ķ×Ķ +à¹Ĥ ล +ĠاÙĦ Ùĩ +ว า +หล าย +Ñī е +à¸Ĥ à¹īà¸Ń +à¹īà¸Ń ย +ب Ø· +ка Ñı +ĠØ ¢ +Ġи Ñģ +ĠاÙĦ غ +à¸ģ า +à¸Ļ à¹Īา +ÙĬ ÙĪ +×ij ×ķר +á»ħ n +ว à¸ĩ +×Ļ× ĸ +ì² Ń +н им +ëŁ ° +×Ĵ ×ķר +ص ØŃ +ÙĦ ÙĪ +×Ĺ ×ķת +ส ุ +رÙĬ ÙĤ +ס ×ĺ +Ġ×ŀ ×¢ +ãĥĨ ãĤ£ +à¸Ħ ิà¸Ķ +ãĤį ãģĨ +à¹Ħ ล +à¸Ļ à¹Į +á»ı i +ÑģÑĤÑĢ о +ส à¸Ķ +ส าร +ÙĪÙĦ Ø© +ầ m +ร à¹Īว +รà¹Īว ม +ร ุ +ĠاÙĦس ÙĬ +ìĺ ģ +Ġ×ŀ ×ij +פ ×ĺ +à¸ķิ à¸Ķ +×ĺ ×Ļ×Ŀ +Ġë ¬´ +ÙĤد Ùħ +Ġdü ÅŁ +ائ ÙĦ +м Ñĭ +ØŃ Ø³ +ÙĪ ص +×Ļק ×Ķ +ãģ§ãģ¯ ãģªãģĦ +à¹Ģ หม +оÑĢ ÑĤ +í Ĩµ +ãģ IJ +к ÑĢа +ีย ว +ع ار +ئ Ø© +íĥ Ģ +ãģ«ãģª ãĤĬ +ج Ø© +ÙĪÙĤ ع +ÑĮ Ñı +×ķצ ×Ķ +ש ×Ŀ +ب ÙĤ +Ġ×Ļ ×Ķ +ÙĬ Ø· +ım ız +д еÑĢж +×Ļש ר×IJ׾ +غ ÙĬر +ร à¸Ńà¸ĩ +à¹Ģรีย à¸Ļ +Ġ×Ķ ×ĺ +หม าย +Ùħ Ùĩ +اÙģ Ø© +Ġо ÑĢг +ÙĪ Ùī +ãĥ© ãĤ¤ +×ŀ ׳×Ķ +ĠÄij o +Ġг оÑĢ +اÙħ Ø© +æ¥ ½ +Ø« ÙĬر +à¸ģิ à¸Ī +á»ĵ n +ÙĨ ب +ÑĢÑĥ д +ìĹ Ī +Ġ×Ĺ ×ijר +ÑĢаР¶ +ạ ch +ت ÙĪ +à¹Ĥ ม +×ij ×Ļ×ij +Ġí Ĩµ +aca ģı +جÙĦ س +à¹Ģà¸Ľ ล +ว à¸Ķ +à¸Ń ล +ãģŁ ãĤĬ +à¸Ľ ัà¸į +Ġìķ Į +عر Ùģ +à¹Ħ à¸Ł +Ø£ Ø® +å¤ļ ãģĦ +à¸Ķ ัà¸ĩ +Ø´ Ùģ +ãģ£ãģ¦ ãģĦãģ¾ãģĻ +׼ ×ł×¡ +ÑĨ е +еÑģ п +Ùħ اÙħ +à¸ŀื à¹īà¸Ļ +иÑĩеÑģ ки +Ø® د +Ùĥ ÙĪÙħ +Ġ×Ķ ר×IJש +ت اب +é£Ł ãģ¹ +ื à¸Ļ +оÑĢ о +Ġb öl +×ķ×Ĺ ×ĵ +دÙĬ ر +ắ m +د ع +ãģķ ãģĽ +à¸ĺ ร +à¸ĺร รม +ãģĭ ãĤĤ +å¤ļ ãģı +r ä +س ع +×Ļ׾ ×Ķ +ض ر +ĠاÙĦ شر +×ĸ ×ķר +×¢ ×ijר +ạ m +алÑĮ но +ر ÙĨ +اÙħ ج +׼ ×ļ +d ıģ +д ен +ض ا +ÙĦÙĬ Ùħ +Ġê·¸ 룬 +تÙħ اع +ار ÙĬØ® +à¹Ĥ à¸ķ +ĠÑģ ÑĢед +Ġ׳ ×ķס +ÙĤ بÙĦ +оÑĤ ов +le ÅŁtir +Ġм еÑģÑĤ +سÙĦ Ùħ +Ġ×¢ צ +ĠاÙĦس ÙĦ +еÑĤ ÑĮ +اب Ø© +н ак +สà¸ĸ าà¸Ļ +Ġ×ij ׳ +à¸ļ ัà¸Ļ +׼ ׳ +Ġö ÄŁ +ãģ¨ è¨Ģ +uy ến +di ÄŁ +áºŃ u +ÑĢ аÑģ +ãĤ· ãĥ§ãĥ³ +n ız +×ķ×ĵ ×Ķ +ت س +Ùħ اÙĦ +à¹Ģห à¸ķุ +ย ว +à¸ŀ ัà¸ģ +ãģĦ ãģªãģĦ +Ġк аÑĩ +ล à¹Į +ר׼ ת +ÅŁt ur +×ŀ ×ķס +ãģ ¥ +б ол +عÙħ اÙĦ +×ķר ת +ÑĨи он +ศ ึà¸ģ +ภı +ÑĢ ен +اس ÙĬ +ائ ر +à¹Ĥ à¸Ľà¸£ +Ġse ç +غ ÙĬ +Ñį ÑĤ +ен н +ãģª ãģ® +×Ļש ×Ķ +×Ļפ ×ķר +ãģŁãĤģ ãģ« +ز Ø© +Ġç oc +ãĤ¯ ãĥª +ÑĪ ен +ãĤı ãģij +رÙĬ د +ĠÑĢ аÑģÑģ +Ùĥ ات +ส à¸Ńà¸ļ +ce ÄŁi +ãĤ¿ ãĤ¤ +à¸ļ ร +ĠاÙĦ بر +׳ ×ķ×¢ +r ün +را ض +ศา ส +à¸ķ รà¹Į +ãģį ãģŁ +×ķ׾ ×ĵ +еÑĢ и +íĹ ĺ +ắ p +ت عÙĦ +Ùĥ د +иÑĤелÑĮ но +Ø· Ùģ +Ġав ÑĤом +Ġ×ŀ צ +ÑĪи Ñħ +ات Ùģ +ĠÑħ оÑĤ +Ùİ Ø§ +ãģı ãĤĭ +×Ķ פ +à¹Ĥ à¸Ĺ +à¹ģ à¸ŀ +à¹Ī à¸Ńย +ĠاÙĦÙħ Ø´ +à¸ģาร à¸ĵà¹Į +ани з +×Ķ ׾ +ظ Ùħ +ย ุ +li ÄŁ +à¹Ħ à¸Ĥ +à¸ĸ ืà¸Ń +ö z +ãģij ãģ¦ +à¹Ģ à¸ľ +ุ ม +ãĥĹ ãĥ¬ +Ġ×Ķ×IJ ×Ĺר +خت ÙĦÙģ +à¸ İ +ÙĦا ØŃ +Ġdü zen +צ ×Ķ +س اء +×ķר ×ļ +×ķ×ĵ ×Ļ +ÑĢа ÑĦ +ÅŁt ır +ãģ« åħ¥ +ãģĪ ãģ° +ص ÙĪÙĦ +ĠÐľ оÑģ +ا Ùĩر +ãģ£ ãģ +ĠлÑİ Ð± +×Ļ×¢ ×Ķ +Ġ×Ķ×ŀ ק +สิ à¸Ĺ +สิà¸Ĺ à¸ĺิ +×Ļ׳ ×Ŀ +ÙĦا Ùģ +à¸ŀัà¸Ļ à¸ĺ +×ķ×IJ ×Ķ +ม ั +à¸Ĥ à¸ĵะ +д оÑĢ +ãģ¨ ãģª +à¸ģระ à¸Ĺ +ac ı +×ķ׾ ×ķ×Ĵ +Ñĥ ÑĪ +ãĥ¥ ãĥ¼ +ãĥ ¦ +Ùħ ست +Ġa ÅŁ +ש ק +פ ת×Ĺ +าย à¸Ļ +í ĩ +ë ¢ +ï · +í ī +ì µ +ì ¬ +ðĿ Ľ +ì Ĵ +ë Ļ +ê § +á ĸ +â ¨ +â ± +á ĺ +ð ĸ +à ł +á Ķ +ðIJ Ń +ữ ng +Å© ng +Ġ×Ķ ת +ĠاÙĦ ا +Ġ×ŀ ת +à¸ĸ ึà¸ĩ +ò n +á»ĭ nh +нÑĭ м +Ġc ả +à¸Ķ ู +Ġ à¹ģà¸ķà¹Ī +Ġ×ij ×Ķ +ó i +ãģ¨ ãģĹãģ¦ +ú ng +ĠØ ° +Ġ×Ķ ׳ +Ġب ÙĨ +ÙĦ اÙĦ +à¹Ħ à¸Ĺย +á»ĩ p +t ı +ม ัà¸Ļ +ằ ng +á»ij t +к ом +à¸ĭ ึà¹Īà¸ĩ +à¸Ħร ัà¸ļ +à¸ļ à¹īาà¸Ļ +ĠاÙĦ ÙĬ +l ü +ÙĪ س +ãģł ãģ£ãģŁ +à¹Ģ à¸ĩ +Ġê³ µ +н Ñĥ +ãĤĪ ãĤĬ +м Ñĥ +à¹Ģà¸Ĥ า +ãĤ Ģ +ни е +ãģ«ãģª ãĤĭ +áºŃ y +ĠÙĪ ا +ëł ¤ +ש ×ķ +á p +×ĵ ×ķ +ãģ§ ãģĹãģŁ +ع ض +Ñģк ой +æĦŁ ãģĺ +ÑİÑĤ ÑģÑı +Ġ×Ļ ׼×ķ׾ +ãĤĵ ãģł +в и +à¹Ģล à¹Īà¸Ļ +ìĿ´ ëĭ¤ +ĠÙĦ Ùĩ +à¸Ħ ืà¸Ń +ت Ùĥ +Ùħ ÙĥÙĨ +a ģı +׳ ×ĵ +ë¯ ¼ +à¹Ħ ว +สำ ห +สำห รัà¸ļ +Ñģл ед +t ır +ĠÙĦ ÙĬ +ĠاÙĦع ÙħÙĦ +×ij ×ķת +×ij ×Ļ×Ŀ +à¸Ħ ำ +à¹Ģà¸Ħร ืà¹Īà¸Ńà¸ĩ +lı ģı +ืà¸Ń à¸ĩ +ج د +íŀ Ī +ìĭ ¬ +×¢ ×ķת +ส ิà¸Ļ +Ñĩ и +ر ض +à¹Ģà¸Ľ ิà¸Ķ +à¸Ħ à¹Īา +ìĦ ł +ÙĪر Ø© +ק ×ĺ +ìľ ł +ع ÙħÙĦ +×IJ ×Ļ×Ŀ +׾ ×Ļ×Ŀ +à¹ĥห à¸į +à¹ĥหà¸į à¹Ī +ừ a +á»į i +ãģ ¶ +ÃŃ ch +ãĥĩ ãĤ£ +×ķר ×Ļ×Ŀ +Ñģ о +ìķ ½ +ов а +Ñĩ аÑģÑĤ +à¹Ģà¸Ī à¹īา +п ÑĢо +Ġ×ŀ ×Ĺ +ãĥ İ +×ķ×Ļ ×ķת +Ġд е +ë§ Ī +ì§ ģ +×Ļפ ×Ķ +ĠاÙĦع اÙĦÙħ +ë¥ ´ +ר×IJ ×Ķ +uy á»ĥn +×¢ ×Ļ +ม ืà¸Ń +Ø¥ ÙĨ +ร ู +ĠØ ² +×Ļ ×ķ×Ŀ +à¸ķ à¹īà¸Ļ +ãģ¦ ãģĦãģ¾ãģĻ +Ùħ اÙĨ +ĠÐ ¥ +à¸Ľà¸£à¸° à¹Ģà¸Ĺศ +á» ³ +׾ ×ij +à¹Ģà¸Ķ à¹ĩ +ãģŁ ãģ¡ +à¸Ĺี ม +à¸Ļ ะ +ìĹ ° +Ġìł Ģ +ÙĦ Ùĩ +ợ i +ĠاÙĦ ز +د ار +ãĤ³ ãĥ³ +м ин +à¹ģห à¹Īà¸ĩ +à¸Ķ ัà¸ļ +׼ ר +ж а +íĸ Ī +×ŀ ×ĸ +ợ i +à¸Ķ า +Ġع بد +à¹ģ ร +×IJת ר +×¢ ׳×Ļ +à¹Ģ à¸Ħ +×ķצ ר +ì§Ģ ë§Į +ائ Ùħ +Ø£ س +uy á»ģn +Ġ×IJ ׳ +׊׳×ķ +×ĸ ×Ļ +ร à¹īาà¸Ļ +ĠÐł оÑģ +ĠÐłÐ¾Ñģ Ñģ +رب ÙĬØ© +t ür +ãĤĭ ãģĵãģ¨ +ظ ر +б Ñĭ +à¸Ĺีà¹Ī สุà¸Ķ +Ġצ ר +èĩª åĪĨ +л аÑģ +ĠÑı в +ĠÑıв лÑı +à¸ŀร à¹īà¸Ńม +à¸Ńา à¸Ī +à¸ļริ à¸ģาร +Ġç ı +ëį ĺ +ĠاÙĦÙħ ست +ت Ø´ +ש ×ķ×ij +ãĤ ´ +Ġyap ıl +ĠاÙĦ Ø° +ุ à¹Īม +à¸ĸ à¹īา +ìĦ ¤ +ì° ¨ +в аÑĢ +à¹Ģà¸ŀ ิà¹Īม +Æ°á»Ľ i +Ùĥ س +à¸Ńย าà¸ģ +ãģ¦ ãĤĤ +Ġг од +ÙĬ ار +à¸ķ à¸Ńà¸Ļ +Ġиг ÑĢ +à¹Ħà¸Ķà¹ī รัà¸ļ +ĠاÙĦÙħ ر +ÙĤ ت +Ġë ĺ +Ġëĺ IJ +ẩ n +ãģĻãĤĭ ãģĵãģ¨ +×Ĵ ×Ŀ +Ġ×ij ×ij +ت د +ÙĪ ار +ãĤ ® +п ол +Ġм ог +تر Ùĥ +ÙĪ Ø« +Ġç ık +ا Ø© +à¹Ģà¸Ķ ียว +มี à¸Ħวาม +Ġ×ŀ ×Ĵ +ص Ùģ +ĠТ ак +Ġ׼ ת +×Ļ×ĵ ×Ļ +ов оÑĢ +ầ y +สิ à¹Īà¸ĩ +ب ت +ür ü +ÙĨ ج +หล ัà¸ģ +×Ļ×Ķ ×Ŀ +ÙĤ ص +з Ñĭ +×Ľ×ª ×ij +Æ° u +m ız +ĠìĦ ¸ +л ог +Ùħ ÙĬÙĦ +ÙĬ ج +íĴ Ī +à¸ŀ à¸ļ +ห ัว +з на +ר ק +à¹Ĥ ร +Ġ×ij ס +ĠBaÅŁ kan +ĠëĶ ° +à¸Ń ัà¸Ļ +ีà¹Īย ว +н еÑģ +à¹Ģà¸Ķ ิà¸Ļ +ÙĬ اÙĨ +×ķ׾ ×Ļ +ا خت +צ ×ķת +ãģĵ ãģĵ +ĠاÙĦ اÙĨ +ĠпÑĢо ÑĨ +ãģ¾ ãģł +׼ ס +ĠاÙĦ Ø¢ +ÙĬ ز +ĠاÙĦد ÙĪÙĦ +Ġíķĺ ëĤĺ +ض ع +ê» ĺ +ÅĽ wi +ย ิ +ãģ¡ãĤĥ ãĤĵ +ĠÙħ Ø´ +à¸ĺ ี +ãģ¨ ãģį +׳×Ļ ×ķת +Ġë ¯ +Ġë¯ ¸ +Ġs ı +ëĭĪ ê¹Į +Ġп л +غ ÙĦ +à¹ģ รà¸ĩ +ب ÙĬر +ãģĤãĤĬ ãģ¾ãģĽãĤĵ +ê· ¼ +Ġy üz +ĠdeÄŁ er +åł´ åIJĪ +á» ¡ +м аÑĤ +รา à¸Ĭ +ÙĪر ÙĬ +ж ен +ãģ¾ ãĤĬ +ãģ® ä¸Ń +×Ļ×ĵ ×¢ +à¸Ń ุ +à¸ļ à¸Ńล +à¸Ľà¸±à¸į หา +ز Ùħ +ÄŁ a +à¸Ń ืà¹Ī +à¸Ńืà¹Ī à¸Ļ +п л +Ġне обÑħодим +׼ ×ij +à¹Ģ ศ +קר ×Ķ +ì² ĺ +ëł ¨ +×ŀק ×ķ×Ŀ +jÄħ c +Ùĩ ÙĦ +Ġ×¢ ×ij×ķ×ĵ +à¹Ħม à¹ī +à¸ģล ัà¸ļ +×ķ׼ ׾ +ק ×ĵ +اÙĦ ÙĬØ© +ر Ùĩ +ãģij ãĤĮãģ° +ĠÙĨ Ùģس +ãĤ¢ ãĥ« +ìĹ Īëĭ¤ +ק ×ķר +н еÑĢ +ب اب +ãĤ ¶ +سب ب +ÙĦ ÙĬÙĦ +ص ÙĨ +ص در +ế m +à¸Ĭà¹Īว à¸ĩ +ØŃ ÙĨ +Ġ×ij ×Ĵ +×ŀ ×ķ×¢ +׾ ×Ĺ +大 ãģį +ت ب +н еÑĤ +×Ļ×ij ×Ķ +б л +ãĥĹ ãĥª +اص Ø© +ãģ¤ ãģij +×Ļ×ŀ ×ķש +ãģĮ ãģĤ +ëĭ ´ +ãģĭãĤĤ ãģĹ +ãģĭãĤĤãģĹ ãĤĮ +ãģ¡ ãĤī +×ij ×ĺ +Ġba ÄŁ +×Ļ×Ĺ ×¡ +×ij ×ķ×¢ +ล ี +פע ×Ļ׾ +им и +g ÅĤ +Ġим е +خد اÙħ +×IJ ×Ļר +Ġy apt +ãģ¨ ãģĦ +à¸ĩ à¹Īาย +׾×Ļ ×ķ +ØŃد Ø« +را ÙĤ +ĠÄIJ i +اد ر +ãģĵãģ¨ ãĤĤ +×ij ×Ļר +Ġв з +ض اÙģ +ת ×ķ׼ +ÑĢ ом +ر ات +à¹Ģà¸Ĺ à¹Īา +ãģĺ ãĤĥ +ãģĿ ãģĵ +اج تÙħاع +à¹īà¸Ń à¸Ļ +ÙĤ Ùħ +ë³ ¸ +Ä ŀ +ש ×Ļ×ķ +×ij ׳×Ļ +ìľĦ ìĽIJ +à¹ģ à¸Ī +×Ĺ ×ķר +دÙĬ ÙĨØ© +ت Ø· +ằ m +ò a +ย à¸Ńà¸Ķ +Ġëĭ ¹ +สุ à¸Ĥ +×ĵר ×ļ +د ÙĨ +س ÙĬÙĨ +ÙĪÙĤ Ùģ +ÑĨ Ñĭ +г оÑĤов +еж дÑĥ +à¸ŀ วà¸ģ +اÙĤ تص +اÙĤتص اد +cz ÄĻ +ni ÄĻ +ÑĢ еб +ØŃ ÙĪ +à¸Ĺ à¹Į +ãĤĪ ãģŃ +д ж +à¸ģล à¹Īาว +دÙĬ Ø« +ãĤ³ ãĥŁ +ÙĤ ÙĪÙħ +Ġت ØŃ +à¹Ģ à¸ķิ +اÙģ ظ +à¸Ī ุ +رÙĬ اض +×ŀש ×ļ +à¹Ĥ ย +еÑĢ е +ãģ¿ ãģŁãģĦ +ìĿ´ ëĿ¼ +ĠاÙĦÙħ ÙĪ +ĠÑģÑĤ о +à¹Ģรà¹ĩ ว +Ġд еÑĤ +ĠÑģ дел +à¹Ģà¸Ĭ ืà¹Īà¸Ń +פ ׳×Ļ +ÙĪض ÙĪع +×ij ס +à¹ģ à¸Ķ +ó c +ริ ม +ÑĢаР´ +ìĪ ł +ãĥ¼ãĤ º +ãģ« ãģĬ +и но +פ ×Ļ׾ +à¸Ĭั à¹Īà¸Ļ +×Ĺ×ĵ ש +à¹Ģà¸Ļ ืà¹Īà¸Ńà¸ĩ +׳ ×Ļס +غ رب +ãĤ¸ ãĥ£ +ส ัà¸ĩ +à¹Ģ à¸Ĺีà¹Ī +à¹Ģà¸Ĺีà¹Ī ยว +ëŁ ¼ +à¹ģ à¸Ł +ãĥ¼ãĤ · +ãĥ¼ãĤ· ãĥ§ãĥ³ +Ġвоз мож +جÙħ ÙĪع +×ijר ×Ļ×Ŀ +ãĥĪ ãĥ© +ĠкаÑĩ еÑģÑĤв +Ø· ÙĬ +ÑĤ Ñı +צ ×ķ×¢ +ÄŁ ını +ع ÙĦÙī +ا Ø° +ÙĪاÙĤ ع +Ùħ ÙĪا +ائ ÙĬÙĦ +к ол +á»ģ m +à¸ľà¸¥ ิà¸ķ +×Ļ׳ ×ĺר +س Ùĥ +ש ×Ļר +ศึà¸ģ ษา +à¸ļ ั +Ñĩ аÑģ +×ķפ ×Ķ +×Ļפ ×ķ׾ +ĠاÙĦس اب +رÙĬ ب +ĠاÙĦ بÙĬ +ãĤ¹ ãĥĨ +Ñĩ ен +à¹ģ à¸ľ +Ġ׳ ש +ز ÙĬد +ØŃ Ø§Ø¯ +ëį Ķ +رÙĪ ع +à¸Ĺุ à¸Ļ +ส มา +c zeÅĦ +×Ļ×ĵ ×Ķ +ãģ§ ãģĤ +Ġçoc uk +Ø® ب +à¸ļ าย +à¸Ľà¸£à¸° à¸Ĭา +×ŀש ׾ +ãģª ãģĭ +à¸ģ าย +ãĥģ ãĥ£ +аÑĢ и +ĠÑĩ а +à¸Ķ ำ +à¸Ĺั à¹Īว +Ñĥ Ñħ +Ġö z +Ġì¢ ĭ +ج رÙĬ +ائ ÙĤ +à¸ł ัย +Ø· ار +د ارة +Ä© nh +Ø« ÙĨ +zell ik +اÙĦ ت +Ġg eli +ãĥķãĤ © +ол од +رب ع +שת ×ŀש +à¸ļร ร +íĿ ¬ +Ġü rün +Ġê·¸ ëłĩ +ศาส à¸ķรà¹Į +ãģ ľ +×Ļ×ij ׾ +ĠпÑĢед ÑģÑĤав +سط ÙĬÙĨ +ãĤĴ 使 +Ġпом оÑī +×ķק ר +ãĥ¯ ãĥ¼ +Ġyö net +×Ļק ר +à¸Ĥ า +еÑĢи ал +ØŃ Ùģ +Ġ×Ļ צ +à¸Ĺ ิ +å£ ² +à¸Ļ à¸Ńà¸ģ +×ķ׼ ר +íĻ ľ +ủ y +ĠاÙĦÙĤ ر +×Ļ×ij ×ķת +ÅĽ ni +Ùħ شار +ượ t +ĠÙĦ دÙĬ +ÑĤ ел +ĠØ¥ ÙĦÙĬ +عÙĦ ÙĪÙħ +ìķ ĺ +в иÑĤ +à¸Ħ ะ +yr ı +ãģ¨ ãģ£ãģ¦ +à¹Ģ à¸ī +à¸ĸ าม +ÙĤ ار +عÙĦ اÙħ +ặ ng +Ùħ ÙĴ +×Ļ×ŀ ת +سب Ø© +ãĤ¯ ãĥ© +×ķס ×£ +ĠпÑĢ ин +ãģĦ ãĤį +س اس +عت بر +วิ à¸Ĺย +วิà¸Ĺย า +س Ùĥر +ãĤ· ãĥ§ +ãģ ģ +ัà¸ģ ษ +×ij ×ķ×Ķ +ห ย +ãģ¾ ãĤĮ +ĠоÑĢг аниз +каз ал +ĠÑģв Ñıз +uy ết +ĠпÑĢо из +Ġק ×ĺ +à¹ģà¸ģ à¹ī +п ÑĥÑģ +Ġê·¸ ê²ĥ +ëĬ IJ +л екÑģ +ãĥ¼ãĥ Ĺ +à¸ķ ำ +ת×Ĺ ×Ļ׾ +à¸Ńà¸ĩ à¸Ħà¹Į +Ạµ +׳ צ +Ø£ Ø´ +Ø´ Ùĩ +ย ะ +à¸ģ à¸İ +ĠاÙĦØ¥ سÙĦاÙħ +ед ÑĮ +ãģ² ãģ¨ +ëıĦ ë¡Ŀ +ãģ© ãģ® +Ñĥ в +еÑĩ ение +ĠاÙĦت ج +ãģ« è¡Į +Ġп озв +ãĤı ãĤĬ +ÙĦ اث +íķĺ ìĺĢ +Ġм аÑĢ +Ġkon uÅŁ +ãĥ¬ ãĤ¹ +ãĤĴ æĮģ +ĠоÑģ нов +×Ĺ ×ij +ÙĪج ÙĪد +פ ×ķף +в оÑĢ +Ġн ик +ãģĭ ãĤĭ +ÅŁtır ma +×Ļס ×ĺ +Ø£ ÙĦ +ห à¹Į +и она +лÑĮ н +Ġг оÑģ +ĠÐľÐ¾Ñģ к +ÑĢ об +×ķ×IJ ×Ļ +ãģĬãĤĬ ãģ¾ãģĻ +ãģ£ãģ ± +к л +à¸Ļ à¸Ķà¹Į +رÙĬ Ùģ +اس ب +ĠÑĢ еÑĪ +Ġд ол +ãģ¹ ãģį +×Ļ×ij ×ķר +м еÑī +Ġна ÑĪ +à¹ģ à¸Ľà¸¥ +ÑĢ иÑĤ +кÑĥ Ñģ +и ÑĢа +аÑĤ ÑĥÑĢ +ÙĪا صÙĦ +à¹Ģà¸ľ ย +à¸Ń ำ +à¹Ģà¸ģ ิà¸Ļ +غ Ùħ +ãģĻ ãģİ +lı kl +ÅĦ sk +ê² ¬ +×Ļ׼ ×Ķ +׊ש×ij +ÙĪر ÙĬØ© +Ġд ейÑģÑĤв +×Ĺ׾ ×ĺ +Ġ׾ ×ŀ×¢ +צ׾ ×Ļ×Ĺ +еÑĩ а +Ùģ اع +×Ĵ ×Ļ×ĵ +áºŃ m +ÄĻ b +Ø´ ع +ãģı ãĤĬ +à¸ŀ ุ +ед еÑĢ +à¸Ĥ à¸Ļ +à¸Ħ าร +ĠболÑĮ ÑĪ +ãģı ãģªãĤĬ +à¸ĵ า +×ĵ ×ķ×Ĵ +Ġм н +ä¸Ĭ ãģĮ +ç¶ļ ãģį +ฤ ษ +ภĨ +Ø® ÙĬ +à¹Ģà¸Ĺ à¸ŀ +สั ม +à¹Ģส à¸Ļ +à¹Ģสà¸Ļ à¸Ń +ãĥ ´ +Ġи ÑģÑĤ +با شر +ĠÑĥ ÑĢов +×ŀ ×ķ×ĸ +ab ı +wa ż +×ķצ ×IJ×Ķ +ÑĤ веÑĢ +à¸ŀัà¸Ļà¸ĺ à¹Į +׳ ×Ĵ×ĵ +ãĤĭ ãģĵãģ¨ãģĮãģ§ãģį +ĠÑĤÑĢ еб +à¸ģร ุà¸ĩ +ØŃت اج +à¹Ģ à¸Ħล +ã Ĩ +ÄĻ tr +Ġszcz eg +Ġר ש +à¸Ĺ à¸ĺ +Ġн ек +Ġнек оÑĤоÑĢ +в ÑĪ +Ð ¬ +à¹Īว ย +ล ุ +б ÑĢÑı +หม ูà¹Ī +à¹ģ à¸ķà¸ģ +ר׼ ×Ļ×Ŀ +Ġí ĸī +ã i +Ùĥر Ø© +â Ń +í IJ +ã į +á ģ +â ® +â ¥ +ì ® +à ¿ +â ¿ +á Ĥ +á ¤ +â ł +í Ł +ðIJ į +ðIJ ° +ðĿ Ĩ +ðŁ Ī +Ġ×¢ ׾ +Ġع ÙĨ +ĠÙħ ع +Ġ×ĸ ×Ķ +ĠÙħ ا +Ġm Ãł +Ġd ụ +á»ĩ c +а Ñħ +s ı +íķĺ ê³ł +Ġ×ķ ×ij +ĠÐŁ о +×ķת ר +ĠÙĦ Ùħ +Ġ×ķ ׾ +ãģĹãģ¦ ãģĦãĤĭ +Ġ×ŀ ×Ļ +Ġب ÙĬÙĨ +з а +ĠÙĥ اÙĨ +Ġ×Ķ ×Ļ×Ķ +ëħ Ħ +×IJ ×ķ +д и +ĠпеÑĢ е +d ı +Ġ׾ ש +Ġש ×ŀ +ãģĮ ãģĤãĤĭ +ãģĦ ãģĦ +ÑĢ е +ק ×ķ +и ли +м е +ÙĬ ت +ãģ§ ãģĤãĤĭ +Ġв о +à¹ĥ หม +à¹ĥหม à¹Ī +Ġש ×ij +Ġ à¹Ĥà¸Ķย +ÙĬ Ùĩ +ãģ§ãģĻ ãģĮ +ãģ¨ ãģ¯ +ר ×ķ +Ġ à¸ĭึà¹Īà¸ĩ +ãģ§ãģį ãĤĭ +м о +à¹Ģà¸ŀ ืà¹Īà¸Ń +צ ×ķ +×ĺ ×ķ +ìķ Ī +Ġh á»į +à¹Ģà¸ĩ ิà¸Ļ +ĠاÙĦ ب +Ġ มี +ë¬ ¼ +Ñģ е +ëĵ¤ ìĿ´ +Ġë§ IJ +Ġl Ỽ +a ÅĤ +×Ĺ ×ijר +Ġd á»± +ÙĬ Ø« +Ġth á»ĭ +à¸ģà¹Ī à¸Ńà¸Ļ +Ġ×ij ׼׾ +ãģ ¸ +ãģ¨æĢĿ ãģĦãģ¾ãģĻ +ả nh +ย า +Ùģ ا +ส ี +à¸ķ า +ë² ķ +ãĥª ãĥ¼ +รา à¸Ħา +Ġ×ķ ׾×IJ +ãģ¨ ãģĵãĤį +à¹Ģล ืà¸Ń +di ÄŁi +ÙĪ اÙĨ +Ġ׾×Ķ ת +รว ม +פ ×Ļ×Ŀ +à¸ľ ม +ж и +c ı +ÑĢ од +Ġkar ÅŁÄ± +×Ĵ ×ķ +ãģ« ãģ¤ +ãģ«ãģ¤ ãģĦãģ¦ +r Ãł +×Ļ×ķת ר +ĠìĨ Į +ק ×Ķ +ÑģÑĤв о +ãģij ãģ© +g é +à¸Ķ à¹īาà¸Ļ +çļĦ ãģ« +ĠÙĬ ÙħÙĥÙĨ +ìĨ į +ÙĬ Ùĥ +à¹Ħว à¹ī +Ñģки й +ì m +Ġ׾×IJ ×Ĺר +à¸Ńา หาร +Ġà¹Ģ à¸ŀ +รา ะ +ล ูà¸ģ +ÑģÑĤ а +Ġìľ ł +ÙĤ ÙĪÙĦ +б оÑĢ +Ñģк ого +หล ัà¸ĩ +à¸Ĥ à¹Īาว +à¹Ģม ืà¸Ńà¸ĩ +ê° ģ +t Ãł +ÙĬ ÙĬÙĨ +عر ض +ë° © +Ġëı Ļ +Ġà¹Ģ à¸Ľ +Ġà¹Ģà¸Ľ à¹ĩà¸Ļ +ç i +li ÄŁi +ìĹIJ ê²Į +ãĤ¿ ãĥ¼ +Ġ׾ ת +פ ×ķת +à¸Ĥ à¸Ń +ر س +ìł IJ +à¸ľ à¹Īาà¸Ļ +ÑĦ и +ج ÙĨ +ì¢ ħ +Ġ×Ķ פ +Ġn go +á»ĭ a +Ġtá» ķ +Ġê·¸ 리 +à¹Ģม ืà¹Īà¸Ń +Ø° Ùĥر +ìĸ ij +ìĹ Ń +×ĺ ׾ +k ı +Ġع ÙħÙĦ +Ġع ÙĨد +à¸ĭ ืà¹īà¸Ń +Ġê± ° +в е +r ü +à¹Ģ à¸Ńา +ส à¹Į +à¸Ī à¸Ļ +ס ת +Ġgi ả +ãĤĭ ãģ¨ +à¸ģำ ลัà¸ĩ +н ей +à¸Ī ริ +à¸Īริ à¸ĩ +Ġë į +Ġëį Ķ +à¸Ħà¹Ī ะ +ì n +Ġsü re +Ġqu y +à¸ļ าà¸ĩ +åıĸ ãĤĬ +ר ×Ĺ +×ij ת +ãģĮ ãģĤãĤĬãģ¾ãģĻ +ר ש +ìĹIJ ëĬĶ +Ġ×IJ פשר +ay ı +ãģĮ ãĤī +ØŃ Ø¨ +ан Ñģ +س ÙĪ +ĠпÑĢ е +د ÙĪ +ãģ« ãĤĪ +à¹Ģà¸ģ ม +สู à¸ĩ +m akt +makt ad +maktad ır +Ġön em +×Ļ×ŀ ×Ļ×Ŀ +б о +ÙĪ ÙĬØ© +รู à¸Ľ +à¹Ĥล à¸ģ +Ùħ ÙĬع +ÑģÑĤ Ñĥп +à¹Ĥ à¸Ń +دÙĬ ÙĨ +ì¤ ij +ãģĹãģ ı +à¹Ģส ีย +в Ñĭ +Ùħ ت +íĺ Ħ +ãĥIJ ãĥ¼ +ا Ø´ +ק ס +Ġtá» ¥ +ล à¸Ķ +Ùģ Ø© +í ijľ +ر ج +k ÅĤad +ĠÅŁ ey +ĠØ£ Ùħ +Ġà¹Ģ ม +Ġب ÙĦ +Ñģ каÑı +ãģ¨ ãģ® +Ġìĭ ¤ +ấ m +ห à¹īà¸Ńà¸ĩ +à¸Ĭ ม +d ü +Ġç ek +Ġê³ ł +×Ĵ ×ij +à¸Ĭี วิ +à¸Ĭีวิ à¸ķ +Ùģض ÙĦ +ภ¯ +ç ı +Ġب Ø´ +ĠÙĩ ÙĨا +ãģį ãģ¾ãģĹãģŁ +t ü +Ġìĺ ģ +ĠTür k +к ÑĤ +פר ס +ãģ¨ãģĦãģĨ ãģĵãģ¨ +í ĶĦ +à¹ģร à¸ģ +ר ×ķף +Ġar as +×ŀצ ×IJ +Ġtá» ī +س ا +à¸ŀ à¸Ń +ĠاÙĦÙħ ØŃ +ãĥ ¤ +ĠاÙĦ است +Ùģ ÙĨ +×Ļ×ŀ ×Ķ +ر ت +ãģ¨ ãĤĤ +Ġна Ñģ +п ÑĢи +Ġ×Ĺ ×ķ +и ла +ÙĬ Ø´ +Ġgö z +Ġ×ij ׳×Ļ +ım ı +ĠÑĤ еÑħ +Ġh á»Ļ +غ ر +к он +اØŃ Øª +Ġ à¸ŀ +à¸Ń à¸Ńà¸Ļ +à¸Ńà¸Ńà¸Ļ à¹Ħล +à¸Ńà¸Ńà¸Ļà¹Ħล à¸Ļà¹Į +Ñħ о +Ñı в +à¹ģ สà¸Ķ +à¹ģสà¸Ķ à¸ĩ +à¹Ģà¸ŀ ียà¸ĩ +ÑĤ ов +ا ÙĬ +Ġ×Ķ ×ĵ +Ġ×ķ ׼ +ãĤī ãģĦ +×ķפ ף +Ġë ¶Ī +ล à¸Ńà¸ĩ +Ø· اÙĦ +Ġн и +ĠÙħ ست +ế c +Ġש ׼ +ĠëķĮ 문 +วัà¸Ļ à¸Ĺีà¹Ī +×Ļ׾ ×ĵ +ØŃ Ø§ +е ÑĨ +Ġc ứ +×ĵ ×ķר +ĠÙħ ØŃ +ר׼ ×ij +بÙĬ ع +ни и +ĠاÙĦØ£ ÙĪÙĦ +à¸Ħว ร +ãģ¨æĢĿ ãģĨ +ĠС о +ائ ÙĬØ© +ر اء +оÑģ об +Ġب Ø£ÙĨ +×¢ ×ķ×ĵ +ĠÑĤ е +ãģĵ ãģĨ +ÑģÑĤ ÑĢа +ай н +Ġsö z +ت ÙĨا +à¸Ń ิ +ặ p +ĠìķĦ ëĭĪ +íķ Ń +Ġר×IJ ש +Ġ à¹Ħà¸Ķà¹ī +Ġ×Ĵ ×ĵ +Ġס פר +обÑī е +ĠÙĪ Ø¥ +ada ÅŁ +ãģ¡ ãĤĩ +ק ×ķ׾ +ÑĢ ез +ĠdÃ¼ÅŁ ün +Ġ×ij ×IJ×ŀ +Ġìĸ´ ëĸ +ער ×ij +н ее +ĠÑģÑĤÑĢ ан +س اÙĨ +yn ı +ĠاÙĦر ئÙĬس +ãģĹãģ ª +Ġ׳ ת +ãģ«ãģª ãģ£ãģŁ +g ü +åıĹ ãģij +׾ ת +ìł Ī +ëĬĶ ëį° +Ø® ÙĬر +à¸ķà¹īà¸Ńà¸ĩ à¸ģาร +ĠÙĦ Ø£ÙĨ +Ġch á»ĭ +ÙĪ Ø© +à¹ĥ ส +ë¶Ģ íĦ° +íķĺ ë©´ +ữ u +à¹Ģหม ืà¸Ńà¸Ļ +б еÑĢ +ĠìĿ´ ìļ© +ĠÑģ еб +wiÄĻ ks +Ġ׳ ×¢ +ÑĤ ÑĥÑĢ +Ġngh Ä© +ש ×ķ×ĺ +ti ÄŁi +Ġde ÄŁi +×IJ ×ij +Ġ×ŀ ×ŀ +ãĥĹ ãĥŃ +wa ÅĤ +à¸Ī ึà¸ĩ +Ø® دÙħ +×IJ ×Ŀ +Ä±ÅŁ ı +cz Äħ +ר ×ĵ +ĠÑĢ Ñĥб +خر Ùī +ãģ® æĸ¹ +Ġд енÑĮ +×Ĺ ×Ļ×Ŀ +еÑĤ е +ëĤ ľ +×IJ ×Ĵ +×¢ ×ķר +ë³ Ħ +åIJĮ ãģĺ +ãĤ ² +ר ×ļ +×ķש ×IJ +ìľ ¡ +ا Ø® +צ ×Ļ×Ķ +á»± a +ãģĪ ãģ¦ +ש×Ķ ×ķ +ан ÑĤ +ลา à¸Ķ +ин г +ë¡ ł +اع د +ÙĪ سط +Ġв оп +Ġвоп ÑĢоÑģ +Ùħ ÙĬÙĨ +à¸Ħ à¸ĩ +×Ļר ×Ļ×Ŀ +c ów +ê² © +Ġê·¸ 룰 +Ġì§ Ħ +Ġש ׾×Ķ +à¹Ģร ิà¹Īม +à¸Ĭ à¸Ńà¸ļ +д еÑĤ +ÑİÑī иÑħ +à¸ļ à¸Ńà¸ģ +æĢĿ ãģĦ +ع ÙĬد +ס ×ŀ +×Ĵ ×Ļ×¢ +צ ×ĵ +ب ات +ĠëĶ° ëĿ¼ +à¸Ī ัà¸ĩ +ãģłãģij ãģ§ +×¢ ×Ļר +ĠÑĩ ел +ĠÑĩел ов +ĠÑĩелов ек +ãĥĥ ãĥģ +à¹Ģà¸ģ ีà¹Īยว +à¸Ķ ิ +Ġפ ×¢ +×Ļ×ŀ ×Ļ +ë° ĺ +Ø® ار +×ij ×Ļת +×¢ ×Ļ×Ŀ +ü yor +ãĤģ ãģ¦ +к лад +Ġ à¸Īาà¸ģ +à¹Ģà¸Ħ ย +ส à¸Ńà¸ĩ +à¹ģ à¸Ħà¹Ī +ẫ u +หà¸Ļ ัà¸ĩ +ש׾ ×ķ×Ŀ +اÙĨ ÙĬØ© +åĩº ä¼ļ +åĩºä¼ļ ãģĦ +à¸ł าย +à¸ļา à¸Ĺ +à¸Ĭา ว +mu ÅŁ +Ġ׾ק ×ij׾ +ãĤ· ãĥ£ +ĠÄ° ÅŁ +×Ĵ×ĵ ×ķ׾ +ج عÙĦ +ë³ Ģ +ยิ à¹Īà¸ĩ +à¸Ļ าย +à¸Ļ ีà¹Ī +วิ à¸ĺี +ãĤī ãģªãģĦ +ëł Ī +Ġ문 ìłľ +Ġ à¸ģ +à¸Ĺำ à¸ĩาà¸Ļ +à¹Ģว à¹ĩà¸ļ +ÑĦ е +楽 ãģĹ +สำ à¸Ħ +สำà¸Ħ ัà¸į +ر Ùħ +ãģķãĤĮ ãģ¦ +Ġоб ла +ר×IJ ×Ļ +หม à¸Ķ +ÙĨ ÙĬØ© +ли н +Ġe ÄŁ +it im +ëł ¹ +ص اÙĦ +ÅĽ l +à¸ľ ิà¸Ķ +ãĥŀ ãĥ³ +åħ¥ ãĤĮ +à¹Ģà¸ķ à¸Ńรà¹Į +ار ÙĬ +ĠÐ ¦ +d ür +ส วย +ë¦ ½ +رÙĥ Ø© +Ġh ã +×Ļת ×Ķ +à¸Ĥ à¸Ļา +à¸Ĥà¸Ļา à¸Ķ +à¸Īำ à¸Ļ +à¸Īำà¸Ļ วà¸Ļ +ש ×ķק +Ġд ом +ì± ħ +ãģĭ ãģij +פ ×ķ׾ +à¸Ĭ าย +Ñģ моÑĤÑĢ +Ñģл Ñĥж +ש ×IJ׾ +кÑĢÑĭ ÑĤ +Ġìŀ ĺ +é«ĺ ãģĦ +ĠÑĢ Ñĥк +ÙĨ ص +д ав +Æ°á» ¡ +ưỡ ng +ر اÙħ +×Ļ׳ ×Ļ×Ŀ +ãĥ© ãĥ¼ +ëĦ ¤ +Ġت ع +l ke +好 ãģį +æĮģ ãģ¡ +Ġë§ İ +Ġy ük +ĠÑģоÑģÑĤ ав +енÑĤ ÑĢ +pe ÅĤ +à¹Ģà¸Ľà¸¥ ีà¹Īย +à¹Ģà¸Ľà¸¥à¸µà¹Īย à¸Ļ +íı ī +ãĤĦ ãģĻ +×Ĺ ×ĸ +×ijר ×Ķ +ë£ ¨ +ìĶ Ģ +بØŃ Ø« +à¹Ģà¸ķ à¹ĩ +ów i +ب Ùĩ +ãģį ãģ¾ãģĻ +Ġ×¢ ×ŀ +×Ĵ ×ķ׾ +ез д +ÙĬÙģ Ø© +สà¸Ļ à¹ĥà¸Ī +Ġת ׾ +Ñı Ñī +Ġس ÙĨ +ĠÙĪا ØŃد +ĠÑģ м +lad ı +ı ld +×Ļר ת +ีย à¸Ļ +ת×Ĺ ×ª +Ġж из +à¸ŀ ั +à¸ŀั à¸Ĵ +à¸ŀัà¸Ĵ à¸Ļา +à¸Ĭ ิ +ا Ø®ÙĦ +ãģ£ãģ¦ ãģĦãģŁ +รั à¸IJ +ãĤģ ãĤĭ +à¹Ĥ à¸ģ +ĠT á»ķ +Ġh akk +ر Ùģ +ìł Ģ +Ñģ об +ãģª ãģijãĤĮãģ° +Ùĩ ÙĪ +Ġë² ķ +ãĤ Ĩ +ĠاÙĦس عÙĪد +Ġ×IJ תר +Ø§Ø º +Ġ׾ ×ĵ +à¹ģ à¸ķ +à¹ģà¸ķ à¹Īà¸ĩ +íĮ Į +Ñĥп иÑĤÑĮ +à¸ŀืà¹īà¸Ļ à¸Ĺีà¹Ī +×ij ת×Ļ +à¹ĩ à¸ģ +ÅĤ at +Ġê°ľ ìĿ¸ +ìłķ ë³´ +ÑĤ ал +Ġgü ven +ĠÄ° l +Ġê° ģ +Ġب ت +×ŀ ×ķ׳×Ķ +ĠاÙĦØŃ ÙĥÙĪÙħ +ÙĤ ات +à¹ģ à¸ģà¹Ī +ห าà¸ģ +н ÑĮ +à¸Ľ รัà¸ļ +มา à¸ĵ +Ġне Ñģк +ĠØ ¶ +สม ั +สมั à¸Ħร +ãģĮ ãģĤãĤĬ +м еÑģÑĤ +Ġ×IJ צ׾ +Ġкомп ани +ס ר +ÙĬÙħ Ø© +ĠÑħ оÑĢо +ĠÑħоÑĢо ÑĪ +Ġ×Ļ ×ķ×ĵ +ü s +×Ĵ ×Ļש +à¸ļ à¸Ĺ +تÙĨ ظ +ว าà¸ĩ +ม หา +Ġ׼ ×ķ׾ +à¸Ĥ à¹īาà¸ĩ +ë° ľ +г од +д ан +ãģĭãĤĤãģĹãĤĮ ãģ¾ãģĽãĤĵ +ãģĵ ãģ¡ãĤī +ãĥIJ ãĤ¤ +ece ÄŁi +دÙĬ دة +ÙĨ Ùī +Ġëĭ¤ ìĿĮ +ว ี +غ ا +ли з +à¹Ģà¸Ķ ิ +à¹Ģà¸Ķิ ม +ĠÙĬ ست +Ġy ılı +ko ÅĦ +ãģ§ãģĹãĤĩãģĨ ãģĭ +ãģĤ ãģª +ãģĤãģª ãģŁ +ÑĨ ен +ĠÙĪ ز +×IJ ×Ļש +à¹Ī à¸Ń +ر ØŃ +ê´ ij +ÑĢа ÑģÑĤ +Ġ×Ķ ׾ +ãģĹãģ¦ ãĤĤ +×ŀר ׼ +×ŀר׼ ×ĸ +éģķ ãģĦ +ãģŁ ãģı +ĠÑģ Ñĥд +в еÑģÑĤи +ĠíķĦ ìļĶ +ãĥķ ãĤ§ +ÑĤелÑĮ но +à¹Ģà¸ŀ ืà¹Īà¸Ńà¸Ļ +ÅĤu ż +à¹Ģà¸Ķิà¸Ļ à¸Ĺาà¸ĩ +ש ×ķר +Ġ×ŀ ×ĵ +×ķ×¢ ׾ +ÙĦ اÙħ +à¹Ħ à¸ĭ +л ей +кÑĥ ÑĢ +Ạ¢ +à¸Ĺ าà¸Ļ +ì§ ij +ĠгоÑĢ од +ר ס +׾ ×ķ×Ĵ +mas ını +Ġл ÑĥÑĩ +ล à¹Īา +ìļ ¸ +ש ×ĺ +ĠÐĺ н +í Ĥ¤ +ÙĪÙĦ ا +ìķ ł +ĠØ£ÙĬ ضا +Ùĥ ار +ĠاÙĦت ع +ส ูà¹Ī +ãĤ ¼ +×ij ×Ļ×IJ +ย à¸ģ +ĠØŃ ÙĤ +ر بÙĬ +ãģĺãĤĥ ãģªãģĦ +รัà¸ģ ษา +Ñħод иÑĤ +à¸ķ à¸Ńà¸ļ +׳ ×ĺ×Ļ +ĠاÙĦÙħ ج +تÙħ ع +ов аÑĤÑĮ +ÙĦ ÙĬÙĨ +×Ļ×ŀ ×ķת +Ġm ù +n ÄĻ +Ġد ÙĬ +׼ ש×Ļ×ķ +Ġhi ç +ë ijIJ +ÙĪ اء +ÙĪ Ø· +ĠاÙĦ بÙĦ +à¹ģม à¹ī +ק ×ķת +ÙĪج د +å§ĭ ãĤģ +ÙĬ ئة +Ġë§ ¤ +ص بØŃ +פ ×IJ +г оÑĢ +ס ×Ķ +بÙĬ ÙĤ +ย าà¸ģ +Ġн ад +ÙĬ Ùij +Ġب ÙĪ +ס ×ķר +Ùħ ÙĥاÙĨ +ר ×ij +×Ĵ ×ĸ +צ ת +b ilit +л аг +ĠN go +×IJ ×ķר +à¸ķ à¸Ļ +íĬ ¹ +à¸Ĺีà¹Ī à¸Ķี +à¸Ľà¸£à¸° à¸Īำ +ов ание +ãģĦ ãģ¤ +ãĥĥãĤ¯ ãĤ¹ +åIJĪ ãĤı +åIJĪãĤı ãģĽ +×Ļ׳ ×ķ×Ļ +ạ y +Ø« ÙĤ +ĠпÑĢ об +ĠпÑĢоб лем +ÅŁ eh +ÅŁeh ir +ع ادة +اÙĨ ÙĪÙĨ +à¸ķัว à¹Ģà¸Ńà¸ĩ +ì¶ ķ +ı lan +б ан +ãĥ³ ãĥī +à¸Ī ี +Ġ×Ķש ׳×Ļ +п оÑĤ +×ķ׾ ×Ļ×Ŀ +ล ัà¸ļ +ĠÑį ÑĤи +×ijק ש +ë¹Ħ ìĬ¤ +à¸Ńยà¹Īาà¸ĩ à¹Ħร +×Ļ׾ ×Ļ +à¹ĥà¸Ĭ à¹Ī +ĠاÙĦ ÙĥÙĦ +ãĥļ ãĥ¼ãĤ¸ +ص Ø© +ÑĤи ÑĢ +ãĤĵ ãģ© +зÑĭ к +wy ż +Ùĩ ÙĬ +ĠÙħ ÙĦÙĬ +Ġвид е +ظ اÙħ +دا ÙĪÙĦ +×ŀ ת×Ļ +Ġs ık +à¹Ģà¸ķิ ม +ãĤ¢ ãĤ¤ +ка Ñħ +צ ×Ļ׾ +à¹Ģà¸Ĭ à¹Īà¸Ļ +м аг +маг аз +магаз ин +à¸Ľ ั +à¸Ľà¸± à¸Ī +Ġש ×Ļר×ķת +ีย ม +ãĥĸ ãĥ« +Ġد ÙĪÙĦ +קר ×Ļ×Ŀ +Ùĩ Ùı +ов о +Ġü ret +د ÙĪÙĨ +à¹ģà¸Ļ ว +à¹Ģà¸Ļ ืà¹īà¸Ń +ĠÑĦ оÑĤ +ãĥ ĺ +ãģ¤ ãģĭ +Ñı Ñģ +ĠíķĺëĤĺ ëĭĺ +ائ ع +Ġп лаÑĤ +ìĺ Ī +Ġdost ÄĻp +ÙĪج Ùĩ +Ġ×Ķ ×Ĺ×Ļ +׳ ×Ļק +д ей +í ĽĦ +ı y +بØŃ Ø± +à¹Ģส ริม +Ġ׾ ×Ĵ +Ø°Ùĩ ب +ج ÙĬÙĦ +رÙĥ ز +Ġë ħ +Ġëħ ¸ +פ×Ļ׾ ×ķ +ãģ¾ ãģļ +iri ÅŁ +ĠÙĥ ÙĬÙģ +Ġ×ij צ +Ġêµ IJ +ÑĢоÑģ Ñģ +ĠØ´ ÙĬ +Ġiç er +×Ĵ ×ķ×ij×Ķ +мен но +×¢ ×ij×Ļר +×ķ×ŀ ×Ķ +ãĤī ãģĹãģĦ +ãģ ¼ +Ñī ин +è²· ãģĦ +جÙħÙĪع Ø© +Ġdön em +Ġ×ij ×IJר +в еÑģÑĤ +×ķר ×ķת +س Ùģ +à¹ģà¸Ĺ à¸Ļ +Ġд окÑĥменÑĤ +Ġا ÙĬ +ج اÙĨ +צ×ķ×¢ ×Ļ +ĠоÑģ об +ĠاÙĦÙħ س +ÑĢаР± +à¸ł ู +à¸Ķ าว +л екÑĤ +ع ÙĤ +×ķ×ĵ ×ķת +Ġol u +Ġolu ÅŁtur +ãģ¾ ãģ¾ +ед ин +à¹Ģ à¸Ńà¸ģ +ãĤµ ãĤ¤ +ëĦ Ī +Ø· ÙĨÙĬ +Ø· ÙĤØ© +ĠÐł аз +ÙĦ Ùij +Ñĩ ем +Ġ׾ ×ĺ +สั à¹Īà¸ĩ +سر ائÙĬÙĦ +Ġפר ×ĺ×Ļ +д еÑģÑĮ +Ġ׳ ׼ +اÙĨ ب +ÙĬا Ø© +Ùħ بر +Ġk ı +à¸Ľ à¸ı +à¸Ľà¸ı ิ +à¸ļั à¸ķิ +׳ ת×Ļ +ìĨ ¡ +ر اب +à¹ĥ à¸ķ +à¹ĥà¸ķ à¹ī +×Ļ׳ ת +ÙĪ ÙĬر +Ġ×Ķ×ŀ ×Ļ +ей ÑĩаÑģ +ק ×ķ×ij +در اس +ĠÙħ ÙĤ +رÙĬ ÙĨ +Ø® اص +ãģĬ éĩij +Ġج دا +ãģĨ ãģ¡ +ëħ ¸ +ır ım +æ§ ĺ +ãģ« å¯ +ãģ«å¯ ¾ +ÑĨ ев +Ġv ard +ĠÐIJ н +e ÄŁ +ÑģÑĤв енно +Ð ¨ +س د +à¸ģ ุ +à¹ģà¸ľ à¸Ļ +รูà¹ī ส +รูà¹īส ึà¸ģ +ات ØŃاد +Ñij ÑĤ +×Ĺ ×ķק +ãģĻ ãģIJ +Ø· ÙĦاÙĤ +Ġק ×ķ×ĵ +à¹ĥà¸Ĭ à¹īà¸ĩ +à¹ĥà¸Ĭà¹īà¸ĩ าà¸Ļ +ãĥ¼ãĤ ¿ +Ġs ür +ÑĢ ок +ë³ ij +สมา à¸Ĭ +สมาà¸Ĭ ิà¸ģ +ãĥķ ãĥ¬ +è¾¼ ãģ¿ +ãĤ» ãĥ³ +Ġê°Ģ ì§Ģ +à¸ľ à¹īา +ÑįÑĤ омÑĥ +иÑĤ ел +à¸ł ั +ภij +ãĥĸ ãĥ© +×Ľ×ª ×ķ×ij +׳ ×Ŀ +ен нÑĭе +×¢ ×¨×Ľ×ª +Ġì Ĥ +ĠìĤ ´ +à¸Ĥ à¹īา +׳ ×ķס +ãĥ¬ ãĥĵ +ÑĢ еÑģ +à¹Ģล à¸Ĥ +Ø« اÙĦ +ìĹ Ĩ +ĠÑĩ аÑģÑĤ +า ศ +ãĥª ãĤ¢ +u ç +×Ļ׼ ×ķת +ล à¹īาà¸Ļ +i ë +ãĤ¸ ãĤ§ +à¸Ī à¸Ń +ÙĪ ØŃد +×Ļצ ×ķ×ij +Ġ×ij ש׾ +ок о +ض Ø© +Ø° ر +ĠÑĥ д +Ä° L +×ķצ ×Ļ×Ŀ +×ĸ ×ŀף +à¸Ľ à¸ģ +íķĻ êµIJ +س اÙħ +à¹Ħ à¸Ķ +ละ à¹Ģà¸Ń +ละà¹Ģà¸Ń ีย +ละà¹Ģà¸Ńีย à¸Ķ +ả y +аÑĨи он +ãĤ¹ ãĤ¯ +פ ×ķס +ร à¹Īาà¸ĩ +ен нÑĭй +ع ÙĨ +عÙĦ ÙĨ +ائ Ùģ +d ÄĻ +ؤ ÙĪÙĦ +׾×ķ ×ķ +Ġ×ij ש×ij +ä»Ĭ åĽŀ +ĠاÙĦج ÙĨ +د اد +wa Äĩ +ãĥª ãĥ³ +ĠìŀIJ ìĭł +اÙĨ ÙĬا +ãĥ¡ ãĥª +ÙĦ ÙĪÙĨ +à¸Ĺ à¹Īà¸Ńà¸ĩ +à¸Ĺà¹Īà¸Ńà¸ĩ à¹Ģà¸Ĺีà¹Īยว +اÙģ ÙĬ +Ġли ÑĪ +Ùħ ÙĬØ© +оÑĤ веÑĤ +Ñĩ ин +à Ĭ +ãĥ¡ ãĥ³ +å® Ł +éļĽ ãģ« +ĠÑĢаР¹ +ãĤ¦ ãĥ³ +×Ļר ×ķש +×Ļר×ķש ׾×Ļ×Ŀ +ม ะ +Ġar a +каз аÑĤÑĮ +à¸ķ ัà¸Ķ +ÑĥÑİ ÑĤ +Ġü st +×Ĵ ×ķ×ij +×Ĵ×ķ×ij ×ķת +mal ı +ег од +егод нÑı +اÙģ ÙĤ +à¸Ĭ à¹Īà¸Ńà¸ĩ +Ġö zellik +×Ļצ ×ķר +Ġmi ÄĻd +Ġili ÅŁ +Ġна Ñħод +×¢ ×ĸר +׾ ×Ľ×ª +ÙĨت اج +ĠÑģ ем +à¸Ī à¹Īาย +à¸ķร ว +à¸ķรว à¸Ī +פר ×ķ +à¸Ĥ ัà¸ļ +ãģ ŀ +Ġп ло +к олÑĮ +×ŀ×¢ ×ĺ +íķĺ ìĭľ +jÄħ ce +ÙĨ اÙĨ +ลี à¸ģ +н ÑĥÑĤ +Ġоб ÑĢаз +Ùĥ بر +ĠاÙĦÙĪ Ø·ÙĨ +ãģķãģĽ ãģ¦ +ÙĤ اء +×ŀ×ĵ ×Ļ׳ +y ü +פ ×Ļת +׳ ×ķף +ÙħÙĨ ظ +หà¸Ļ ัà¸ģ +ìŀ Ī +ãĤ« ãĥ¼ãĥī +ع ÙĨÙĬ +п од +ض اء +à¸Ļ à¸ķà¹Į +×ŀש פ +ว à¹Į +ר ×ķק +ส ืà¹Īà¸Ń +פק ×Ļ×ĵ +ãģªãĤī ãģªãģĦ +ĠìŬ 룬 +ÙĦ ج +Ñī иÑĤ +ãĥĥ ãĤ· +ÙĦÙĬ س +ĠÙĦ Ùħا +ìł ij +×ij ×Ļף +ãĥģ ãĤ§ +Ġgü ç +Ġch ứ +×ķצ ×IJ +קר ×ij +à¹Ĥ à¸ŀ +оÑĩ но +סק ×Ļ +ש׾ ×Ŀ +صر Ùģ +ĠL Ãł +×¢ ×Ļת +á» · +à¹Ĥ à¸Ńà¸ģ +à¹Ĥà¸Ńà¸ģ า +à¹Ĥà¸Ńà¸ģา ส +Ġ×Ķ ×ĵ×ijר +à¸Ļั à¹Īà¸Ļ +ز ر +нак о +íļ į +ãĤĤ ãģ¡ +ãĤĤãģ¡ ãĤį +ãĤĤãģ¡ãĤį ãĤĵ +اÙħ ت +عد اد +и нÑĭ +ÅĤy w +à¸Ħ à¸ĵะ +à¸Ĺ ะ +kt ör +×Ļ×Ĺ ×Ķ +Ġм е +Ġме ÑģÑı +׳×Ķ ×Ĵ +ĠÑģ ÑĥÑīеÑģÑĤв +à¸Ļ ัà¸Ļ +ÑĦ ÑĦ +ек ÑĤив +عÙĦÙĪÙħ ات +б Ñĥд +à¸Ļัà¸ģ à¸ĩาà¸Ļ +หà¸Ļà¹īา à¸Ĺีà¹Ī +ÙĤÙĬ ÙĤ +ãĤ· ãĥ³ +ãģ« éĸ¢ +×IJר ×Ĵ +ĠпÑĢ оÑĤ +ĠпÑĢоÑĤ ив +ĠìŀĪ ìĸ´ +ÙĤÙĬ ÙĤØ© +ìĹ ĩ +k ür +ãģ«ãģªãĤĬ ãģ¾ãģĹãģŁ +Ġде ÑıÑĤ +ĠдеÑıÑĤ елÑĮ +פ×ķר ×ĺ +à¸Ł à¹īา +à¹Ģ à¸ł +ĠавÑĤом аÑĤ +×ĸ ×Ļק +Ġold uk +ع اÙħ +ĠÑĤ оÑĢ +yrı ca +ê Ì +ãĤŃ ãĥ³ãĤ° +ãģ« ãģ¨ãģ£ãģ¦ +à¹Ģà¸ī à¸ŀ +à¹Ģà¸īà¸ŀ าะ +ãģ¯ ãģļ +×ŀ ×IJ×Ļ +สะ à¸Ķ +สะà¸Ķ วà¸ģ +ìľ¼ ë©° +à¸ģ ี +ภ¬ +Ġ×¢ ×ķש +à¸łà¸² ษา +à¸Ĺ ัà¸Ļ +ac akt +acakt ır +اع دة +ĠÑĥÑģл Ñĥг +ס ר×ĺ +×ķ×ŀ ×ķת +×Ķ ×ķר +×ŀ ×ķ×ij +×ŀ×ķ×ij ף +سÙĬ اس +اتÙģ اÙĤ +×Ķ צ׾ +Ùħؤ س +Ġp ó +Ġк ни +×Ļ׼ ×ķ׾ +à¹Ģหล ืà¸Ń +׼׾ ׼ +׳ ×ĸ +ÑĪи е +r ès +ĠاÙĦØŃ ÙĤ +лÑı ÑĢ +ห à¸į +หà¸į ิà¸ĩ +ר×Ĵ ×Ļש +à¹Ģส à¹īà¸Ļ +ש×ij ×ķף +ô tel +ап ÑĢ +апÑĢ имеÑĢ +اب ÙĦ +ĠÑĢаз виÑĤ +Ġп олÑĮз +ĠС еÑĢ +×ķ×ij ×Ļ +r óż +ìĭ Ń +ãĤ¯ ãĥĪ +ãģĹ ãĤĪãģĨ +à¸ģร ม +ØŃ ÙĥÙĪÙħ +à¹Ĥ à¸ļ +à¸Ĺ à¹īาย +ĠM á +ĠÑĤ Ñĭ +à¸Ħร ัว +ÑĢÑĥ б +ạ p +Ġm ÅĤ +ĠmÅĤ od +Ġgör Ã¼ÅŁ +Ġgeli ÅŁ +Æ°Æ¡ i +×ŀש ק +ÙĢÙĢ ÙĢÙĢ +รา ว +ãģĹãģ £ +ãģĹãģ£ ãģĭãĤĬ +ĠÐļ он +Ġk ê +à¹Ĥà¸Ĺ ร +èIJ½ ãģ¡ +åĩº ãģ¦ +ล ัà¸ģษ +Ġ×Ĵ ×ij×ķ×Ķ +ãĥĻ ãĥ« +ê±° ëĤĺ +ë§ IJ +×Ļ׾ ×ĵ×Ļ×Ŀ +ĠëĦ Ī +×ŀר ×Ļ +ร ส +ãĥŃ ãĥ³ +и ло +ноÑģÑĤÑĮ Ñİ +×ĸר ×Ĺ +п он +Ġ×Ķש ׾ +ê²ł ìĬµëĭĪëĭ¤ +Ġki ÅŁ +ĠÐļ и +ว ร +د اع +ÅŁ im +ÙĨ Ùij +в аÑĤ +را Ùĥ +ب اÙĦ +ид е +Ġ×Ķ×ŀ ×Ĺ +ìĸ µ +تÙģ اع +Ø£ ت +ëĬ ĺ +ש ×Ļת +ست Ùħر +ĠÑĦ ак +ĠاÙĦØ£Ùħ رÙĬ +ëŀ ¨ +اس Ùħ +Ġa ÄŁ +Ġç ev +Ùĥ ÙĪر +ãģķ ãģ¾ +Ġç öz +Ġر س +Äħ da +สà¸Ļ ุ +ãģĹãģ¦ ãģıãĤĮ +н Ñİ +leÅŁ me +ãĤª ãĥ³ +ãģ¨ ãģªãĤĬ +ava ÅŁ +×ĺ ×Ļ×ij +ØŃ Ø¶ +×ķצ ×IJ×ķת +ÙĨ ÙħÙĪ +ı t +ĠÑħ а +ĠÑħа ÑĢак +ĠÑħаÑĢак ÑĤеÑĢ +Ġd ÅĤ +ãĥĹ ãĥ© +à¸Ĭ ุม +à¹Ī à¸Ńà¸Ļ +×ķ×ij ׾ +Ñģ ол +×ĵ ×Ĵ +аÑĢ аÑĤ +n ivers +Ġgerçek leÅŁtir +ĠاÙĦ ÙĦÙĬ +ระ ยะ +ĠÙħ ختÙĦÙģ +Ġgö nder +Ùģ ار +do ÄŁ +doÄŁ an +ص ÙĦاØŃ +Ġyay ın +ãĥĨ ãĥ³ +รว à¸Ī +×Ļ×Ĺ ×Ļ×ĵ +ünk ü +ÑĨи алÑĮн +à¸ļ ู +ม ุ +h ä +Ø® Ùģ +å¢ Ĺ +å¢Ĺ ãģĪ +еÑĩ но +ĠاÙĦس ÙĨ +à¸Ĥ าว +im di +Ð « +à¸Ļà¸Ńà¸ģ à¸Īาà¸ģ +à¸ļา ล +ת ש +Ġdüzen le +мÑĭ Ñģл +ãģı ãģª +ż u +Ġwsp óÅĤ +Ġн аз +ınd aki +تر Ø© +ÅŁ ek +Ġö d +ĠÙĪ Ùĥ +Ġпозв олÑı +Ġת ×ķ׼ +ÙħÙĨ تج +ë§ ī +ĠاÙĦØ« ÙĦاث +аÑĨи Ñİ +ÙĪر ÙĪ +Ñĭв аеÑĤ +خص ص +ĠاÙĦÙģ ÙĦ +ĠاÙĦÙģÙĦ سطÙĬÙĨ +Ø¥ جر +إجر اء +اÙĨت Ø® +اÙĨتخ اب +ار ÙĬØ© +×ķ Ö +Ø¢ ÙĨ +×ŀ×¢ ×ķת +Ġм ал +Ġ×IJ ×Ĺ +à¸Ĺ à¹īà¸Ńà¸ĩ +ze ÅĽ +Ġë§Į ëĵ¤ +رÙĬ ع +äºĭ ãĤĴ +à¸ļริ หาร +׾ ×ŀ×Ļ×ĵ +Ġм Ñĥж +ت رÙĪ +ĠباÙĦ Ø¥ +פ ×Ļק +ز ÙħØ© +ĠÃ¶ÄŁ renc +ãĥ ¶ +اÙħ عة +ק×ij ×ķצ +×ŀ ׳×ķת +رÙĬ Ùħ +Ġо каз +ãģłãģij ãģ© +Ġh ız +Ġש ×IJת +ãĤ¢ ãĥ¼ +Ġmożli wo +ìĦ ¼ +ÙĪ اب +ог ÑĢаÑĦ +Ġعبد اÙĦ +ãĤĴ è¡Į +ب ÙĬÙĦ +ĠÄ° ç +ย าย +ĠÑĥ ÑĩаÑģÑĤ +ÑĦ еÑģÑģ +ÑĦеÑģÑģ иона +Ạ¤ +ÙĨ ÙĬÙĨ +عد ÙĦ +สร ร +دÙĬ ÙĦ +×ij ×Ļק +czy ÅĤ +ÑĢом е +Ġм ед +ìĻ Ķ +ãĥ© ãĤ¤ãĥ³ +ĠÑĤ еп +еÑĢ ÑĮ +i ÄŁi +в ели +ÑĢи ÑģÑĤ +ס ×ķפ +×ŀ׾ ×Ĺ +ĠاÙĦØ¥ ÙĨ +Ġ׾×Ķ ש +è¶Ĭ ãģĹ +ĠÑĢ Ñĭ +×ķ×IJ ר +رÙĩ اب +פ ×ķ×IJ×Ļ +ĠгоÑģ Ñĥд +ĠгоÑģÑĥд аÑĢ +ĠгоÑģÑĥдаÑĢ ÑģÑĤв +ĠاÙĦØ£Ùħ ÙĬر +Ùħ ج +à¹Ģหม าะ +ÑĢ ев +à¸Ĭี à¸ŀ +ãĥķ ãĥĪ +иÑĩ но +ĠاÙĦÙħ ؤ +Ġi ht +íħ ľ +د ÙĨÙĬ +ر ص +ла ÑģÑĤ +à¹Ģหล à¹Īา +ılı r +ร à¸ĵà¹Į +×ŀש ×Ļ×ļ +Ġd á»ĭ +Ø·Ùģ اÙĦ +×ĺ ×ķף +Ġ×ij ×Ļ׳ +ãģ¾ ãģ£ãģŁ +лож ениÑı +تØŃ Ø± +ب اØŃ +à¹Ģส ืà¹īà¸Ń +ãģĻ ãģĶ +lt ür +à¸ĩ าม +Ġt ü +ĠпÑĢ им +ĠпÑĢим ен +Ġhay at +ëĥ IJ +ëĭ Į +׳×Ļ ×ķ +вед ен +ìħ ¨ +à¸Ī ัย +à¸ģà¹Ī à¸Ń +Ġв од +оÑģÑĤ оÑı +н аÑĤ +à¹ģ หล +سÙħ ÙĬ +à¸Ķำ à¹Ģà¸Ļ +à¸Ķำà¹Ģà¸Ļ ิà¸Ļ +w ód +ö yle +ãĥĢ ãĤ¤ +ÑĪи й +меÑī ен +ãģĹãģ¾ ãģĨ +ãĥī ãĥ© +ÙĪض ØŃ +à¸Ńà¸Ļ ุ +ĠاÙĦ اجتÙħاع +laÅŁ ma +à¸Ħ à¸Ńà¸Ļ +×ŀר ×Ļ×Ŀ +ÙĨ اÙħج +שר ×ķת +اÙĦ Ø£ +Ġksi Äħż +Ġа н +ÑĢаР¹ +اÙĩر Ø© +×ŀ×ĵ ×Ķ +ä¸Ģ ç· +ä¸Ģç· Ĵ +ä¸Ģç·Ĵ ãģ« +ÑĢиÑĤ оÑĢ +d ıkl +à¹ģ à¸ĸ +à¹ģà¸Ĥ à¹Īà¸ĩ +екÑĤ оÑĢ +×ŀס ×¢ +ÑĢак ÑĤи +u ÄŁu +×ķ×ij ת +สู à¸ķร +ĠçalÄ±ÅŁ m +ĠçalÄ±ÅŁm alar +Ġа на +ãĥĽ ãĥ¼ãĥł +Ġböl üm +Ġب ص +ол оÑģ +ĠìķĬ ëĬĶ +à¹Ī ะ +ÙĪ تر +ä¹ Ĺ +ست خداÙħ +פ×Ļ ×Ļס +פ×Ļ×Ļס ×ij +פ×Ļ×Ļס×ij ×ķק +Ġк ÑĢаÑģ +ли к +رÙĬ ØŃ +×ŀש ׾×Ķ +à¹Ģย ีà¹Īย +à¹Ģยีà¹Īย ม +в иÑģ +ом н +ÄŁ un +ãĥŃ ãĥ¼ãĥ³ +Ø£ تÙĬ +à¸ķร ี +çĶ³ ãģĹ +تÙħ ر +ìĹ ĪìĬµëĭĪëĭ¤ +ĠÙĪ غÙĬر +red ni +ĠاÙĦص Ùģ +Ġна ÑģÑĤоÑı +ĠнаÑģÑĤоÑı Ñī +à¸ķ รา +ĠÑĥÑģл ов +ĠÑĥÑģлов иÑı +ÑĨ еп +×Ķ ×Ĺ׾×ĺ +Ø· ÙĬع +ĠB akan +ĠاÙĦ رÙĪ +илÑĮ но +Ġм еÑĤ +à¸Ķ à¸Ńà¸ģ +ãģĭãĤī ãģªãģĦ +Ġпо ÑģÑĤоÑı +ĠпоÑģÑĤоÑı н +ĠÑĩ аÑģ +ü c +wr ó +б ÑĥÑĢ +ãĥIJ ãĥĥãĤ¯ +ãĥ©ãĥ³ ãĥī +Ġо гÑĢ +สั à¸į +สัà¸į à¸įา +มั à¹Īà¸Ļ +à¸Ħ à¸Ńม +al ık +Ġн ед +üm üz +ĠÅĽ wie +é rio +×Ļ×IJ ×Ķ +دÙħ ات +ı rl +ĠоÑĤ з +ĠоÑĤз Ñĭв +ä»ĺ ãģį +Ġkaż de +мин иÑģÑĤ +ãĤ° ãĥ« +ë° ĸ +ез н +اÙĦ Ùģ +Ġש ק׾ +Ùħ ض +ãĥĿ ãĥ¼ãĥĪ +ÙħÙĨ ت +ÙĤÙĬ اÙħ +Ø´ ÙĨ +×Ļר ×ķ×¢ +ãĤŃãĥ£ ãĥ³ +доÑĢ ов +×ŀ ×Ļת×Ļ +ÙĪÙĦ ÙĪج +Ùĥ اÙģ +ĠÑĢаз лиÑĩ +иÑĤ еÑĤ +н олог +ลà¸ĩ à¸Ĺุà¸Ļ +Ġyak laÅŁ +ãĥ¬ ãĤ¤ +ê²ł ëĭ¤ +æ±Ĥ ãĤģ +رÙĪ Ùģ +Ġí Ĭ +ĠíĬ ¹ +ãģ£ ãģıãĤĬ +à¸Ħวาม à¸Ħิà¸Ķ +×Ķ ×Ļס×ĺ +Ø¥ ÙĤ +ãģ¦ ãģĦ +à¹Ĥ à¸Ĭ +ĠBü yük +ĠФ едеÑĢ +ÑĨи н +ÑĢов а +ĠاÙĦ اÙĤتصاد +Ġch á +à¸ĺ าà¸Ļ +ë¥ ł +à¹Ħ à¸ķ +ÃŃ pio +Ùĭ ا +Ġоб Ñıз +Ùĩ ج +Ġì¤ij ìļĶ +ãģ® ãģ§ãģ¯ãģªãģĦ +بار اة +ãĤ¤ ãĥ« +Ġн оÑĢм +á»ī nh +m ö +mö glich +ÑĨи п +ãĤ¢ ãĤ¯ +×Ķ ×Ļ +ÑĨи алÑĮно +ĠÅĽ wi +ت ÙĤ +ĠÑģÑĤо им +بÙĬ عÙĬ +Ġ׾ ש×ŀ +г лÑı +глÑı д +ãģ¦ ãģıãĤĮ +ÄĻd zi +à¸Ĥ ั +à¸Ĥั à¹īà¸Ļ +Ø· ÙĤ +ĠìĹ Ń +ãģ£ãģ¦ãģĹãģ¾ ãģĨ +ĠdeÄŁer l +ĠdeÄŁerl endir +Ġü lk +Ġмн ог +๠ĭ +ë¿ IJ +ĠУ кÑĢа +ÄŁ ini +Ġбез оп +Ġбезоп аÑģ +à¸Ńà¸Ńà¸ģ à¹ģà¸ļà¸ļ +Ø§Ø ¸ +ØŃد اث +л еÑĢ +×Ļ× ¥ +×Ļ׳×ĺר ׳×ĺ +lar ınız +ØŃÙĬ ØŃ +ż eli +à¸Ń ัà¸ĩ +à¸Ńัà¸ĩ à¸ģ +à¸Ńัà¸ĩà¸ģ ฤษ +ĠоÑĤ лиÑĩ +ั ส +ëŀ į +ож но +ãĤ¹ ãĥĿ +ĠÑħ оÑĩ +Ġк ап +еÑĩ ен +ØŃÙĦ Ø© +ÙĬا Ùĩ +на л +×ķצ ר×Ļ×Ŀ +Ġk ald +åĥ į +ĠاÙĦØ´ خص +Ġз на +Ġwz gl +ż ycz +ê° Ŀ +à¸ŀ ลัà¸ĩ +íģ ¼ +Ġö l +Ġb ụ +Ø´ Ùĩر +Ġз ам +Ġд ев +×Ļ×ĺ ת +تعÙĦ ÙĤ +ÙĪÙħ Ø© +ãĤĴ ä½ľ +ãģį ãģ¦ +í ĥĿ +ras ında +ãĤĴ æİ¢ +ĠÙħ باشر +راج ع +Ġв озд +ÙħØŃ Ø§ +×ķש ר +ĠиÑģÑĤ оÑĢ +ม ัà¸ģ +t ıģ +Ø« ار +تر ÙĨت +à¹ģà¸Ĥ à¹ĩ +à¹ģà¸Ĥà¹ĩ à¸ĩ +п оÑĩ +Ġ×ij ×IJ×ķת +ë¯ Ģ +ëĿ¼ ëıĦ +à¸Ĭ ัà¸Ķ +ส à¸ķà¹Į +ãĥĭ ãĥĥãĤ¯ +ид енÑĤ +Ġг ÑĢÑĥпп +ت Ø® +Ạł +ย ืà¸Ļ +ย ัà¸Ļ +ó ry +T Ãľ +ãģĹ ãĤĥ +ĠпÑĢов ед +лÑı еÑĤ +Ùħ Ø® +ย à¸Ńม +×Ľ×ł×¡ ת +ĠاÙĦÙħ ÙĨت +Ġol mad +ר׼ ×ĸ×Ļ +Ġв ÑģÑĤÑĢ +ĠиÑģ Ñģлед +ÑĤвеÑĢ ж +بد ÙĪ +еÑĢ ÑĤ +ï» · +± ħ +สัม à¸ŀัà¸Ļà¸ĺà¹Į +ิ à¹Īà¸Ļ +צ ×Ļ×ij +wiÄĻ t +Ġì° ¸ +Ġz wiÄħz +سب ÙĪع +ãĥĥ ãĤ° +à¸Ľà¸¥ à¸Ńà¸Ķ +à¸Ľà¸¥à¸Ńà¸Ķ à¸łà¸±à¸¢ +ãĤĤ ãĤĬ +ÙĤد س +Ġspr z +Ġsprz eda +Ġist edi +Ġk hu +Ġд ен +Ġko ÅĦ +Ġ×ij ×Ĺ×Ļ +à¹Ģà¸Ĺ à¹īา +×ķס ×Ļ×£ +ãĥĭ ãĥ¥ãĥ¼ +ĠпÑĢед оÑģÑĤ +ĠпÑĢедоÑģÑĤ ав +à¹Ĥ à¸Ł +é v +ĠاÙĦص ØŃ +صØŃ Ø§Ø¨ +à¹Ģà¸Ī à¹ĩà¸ļ +вл ек +วั à¸ķ +à¸ĸ ุ +ãģĵãģ¨ãģĮãģ§ãģį ãģ¾ãģĻ +ÙĤÙĬ ÙĤÙĬ +×ķ׊ר +Ñĭ ÑĪ +ĠоÑĤ но +ĠоÑĤно ÑĪ +об илÑĮ +Ùģ ØŃ +ı nt +ınt ı +Ġ׾ ×ij×ĵ +í İĺìĿ´ì§Ģ +ãĥĬ ãĥ« +ĠÙħ ساء +×Ļ×ĺ ×ij +ÑĮ еÑĢ +ëĦ · +Ñĭ ÑĤа +ĠоÑĩ еÑĢ +à¸Ķ ืà¹Ī +à¸Ķืà¹Ī ม +ĠN gh +ت عب +ÙĦاÙĤ ات +×ķ׾×ķ×Ĵ ×Ļ×Ķ +ĠìĿ´ ê²ĥ +Ġ×Ķ ×ijר +ìľ µ +à¹Ģà¸Ħล ืà¹Īà¸Ńà¸Ļ +Ùĩ Ø© +à¸Īำ à¹Ģà¸Ľà¹ĩà¸Ļ +å¤ī ãģĪ +wi ÅĽcie +ch od +chod zÄħ +в ÑĢо +×ŀ×Ĺ ×Ļר +Ġy ı +Ġyı ll +ì¡ Į +à¹Ħ หว +ãģªãģı ãģª +Ġзав иÑģ +ĠìĺĪ ìĪĺ +Ùģ Ø° +ủ ng +à¸ŀุ à¸Ĺà¸ĺ +з н +lay an +ãĤ ¡ +à¸ģà¹ĩ à¸ķาม +ĠsaÄŁ lam +ร à¸ĵ +ĠÑģ иÑĤ +ĠÑģиÑĤ Ñĥ +ĠاÙĦت ÙĨ +×Ķ ×ĸ +ĠØ· ÙĪÙĬÙĦ +ta ÅĤ +Ġgö rd +å¤ī ãĤı +ëĥ ¥ +à¸Ħà¹Ī à¸Ńย +×IJ ×ķ×ĺ +ëħ IJ +ãĥ©ãĥ³ ãĤ¹ +วั à¸Ĵ +วัà¸Ĵ à¸Ļ +Ġol uÅŁ +פע ×ķ׾ +Ġszczeg óÅĤ +à¸Ħา สิ +à¸Ħาสิ à¹Ĥà¸Ļ +pow ied +ĠÑĤ еб +หà¸Ļ à¹Īวย +Ġм ил +ØŃ Ùĥ +à¸Ĺ à¸Ķ +ĠмаÑĤ еÑĢиал +ÅĤ ow +à¹Ģà¸ģ ีย +ĠÑģов еÑĢ +ãĤ © +à¸Ľ ริ +Ġи Ñİ +наÑĩ ен +ÑĢен д +mu ÅŁtur +ĠпÑĢод Ñĥк +з д +Ñı ÑĤи +ÑıÑĤи Ñı +à¹Ģม ีย +رات ÙĬج +Ġam acı +ש ×ķ׾ +ש×ķ׾ ×Ĺ +สะ à¸Ńา +สะà¸Ńา à¸Ķ +פ×Ĵ ×¢ +عب Ø© +d ın +íħ Ķ +Ġ×ŀש ×Ĺק +Ġfi yat +Ġз аÑı +ĠзаÑı в +à¹Ĥ หล +à¹Ĥหล à¸Ķ +à¸ģรุà¸ĩ à¹Ģà¸Ĺà¸ŀ +צ×Ļ ×Ļף +ìļ ± +Ùħ ب +Ùħب اد +land ır +Ġв еÑģÑĮ +Ġh ük +ĠÐĴ оз +ÑĩиÑĤ Ñĭва +ว ล +×ķצ ×¢ +à¸Ĥà¸ĵะ à¸Ĺีà¹Ī +ĠaÅŁ aģı +׾×IJ ×ķ×ŀ×Ļ +tr zym +Ã¤ÃŁ ig +owo ÅĽci +ãģĿ ãĤĤ +Ġroz wiÄħz +ĠgÅĤ ówn +м онÑĤ +×ŀ ×ķ×ŀ +ĠÑģÑĤ ан +ÙĦا ÙĤØ© +p rowad +prowad zi +ĠÑģоÑģÑĤ оÑı +×Ļ×IJ ×ķת +r ı +g ı +ãĥij ãĥij +Ġна лиÑĩ +×Ķ צע +Ġ׳ ×Ķ +à¸Ħ ัà¸ļ +ع راض +и ж +Ùĩ ائÙĬ +ãĤī ãģı +ож еÑĤ +Ġоб оÑĢ +ĠобоÑĢ Ñĥд +Ø£ سÙĦ +à¹ĩ à¸Ķ +ÑĢÑĥ ÑĤ +دÙĬ ÙħÙĤ +دÙĬÙħÙĤ را +Ġjest e +×ķ×ķ ×Ļר +×ij×ĵ ×Ļק +деÑĢж ива +ãģĬ ãģı +ewn ÄĻtr +ewnÄĻtr zn +à¸ŀ ฤ +Ġ×IJ ×ķ×Ķ +ת×Ĺ ×ķש +Ġz ob +д Ñĥм +ĠÑģ Ñĭ +ÙĬر ا +ĠwiÄĻ ks +à¹ģà¸ķà¸ģ à¸ķà¹Īาà¸ĩ +lar aras +lararas ı +íĺ Ģ +ëī ´ +×ķ×Ĵ ׾ +ĠоÑĤ меÑĤ +ĠÑĢ ан +ت ÙĥÙĦ +иÑĤелÑĮ н +à¸Ľà¸£à¸° วั +à¸Ľà¸£à¸°à¸§à¸± à¸ķิ +ìŀ ĸ +мож но +pie czeÅĦ +pieczeÅĦ st +ëª » +ìĬ ¨ +×ŀס ×ŀ +á» ¦ +ศ ิ +ศิ ล +ศิล à¸Ľ +ĠÅļ w +ãĥĥ ãĤ·ãĥ§ãĥ³ +unit Ãł +Ġmiesz ka +Ġmieszka ÅĦ +pr zed +przed si +przedsi ÄĻb +przedsiÄĻb ior +à¸Ľà¸£à¸° สิà¸Ĺà¸ĺิ +à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺิ à¸łà¸²à¸ŀ +ย à¹Ī +ìķ Ļ +รว à¸Ķ +รวà¸Ķ à¹Ģรà¹ĩว +å½ĵ ãģŁãĤĬ +äl le +Ñĥ еÑĤÑģÑı +ã n +ëł µ +th è +ãĤĴ åĪ©çĶ¨ +ì µľ +íĵ ¨ +à¸Ĺ ัà¸ļ +า à¸Ħม +ãģ ĩ +ëĤ Į +à¹Ģà¸Ľà¸¥ à¹Īา +â ¦ +ë ¾ +ê Ģ +ê ĩ +â ¡ +ðŁ Ł +ã IJ +â º +á Ń +á Ļ +á ĵ +á ² +ðĵ ı +á ¬ +â ¯ +ä ¨ +ê Ŀ +ê « +ð ij +ðĵ ĥ +ðĿ ħ +< unk + + + +Ġع ÙĦÙī +Ġm á»Ļt +Ġv Ỽi +Ġng Æ°á»Ŀi +ĠØ¥ ÙĦÙī +Ġnh ững +Ġth á»ĥ +Ġ×IJ ×ķ +Ġ×¢ ×Ŀ +ا Ùĭ +Ġ à¹ģละ +ĠÙĦ ا +Ġnh Æ° +ĠاÙĦت ÙĬ +Ġ×Ķ ×ķ×IJ +ĠÄij ến +ĠØ£ ÙĪ +Ġv á»ģ +ĠlÃł m +Ġs ẽ +Ġc Å©ng +Ġ ợ +ĠÄij ó +Ġnhi á»ģu +Ġt ại +Ġtr ên +Ġ×Ĵ ×Ŀ +Ġnh Ãł +Ġ׼ ×Ļ +Ġs á»± +ĠÄij ầu +Ġb á»ĭ +ĠÙĩ ذا +Ġnh ất +Ġph ải +Ġhi á»ĩn +Ġdụ ng +ĠÄij á»Ļng +ĠاÙĦÙĦ Ùĩ +ĠØ Į +ĠÙĥ ÙĦ +Ġvi á»ĩc +Ġn Äĥm +Ġth ì +Ġh á»įc +ĠÙĪ ت +t é +Ġا ÙĨ +Ġt ôi +Ġ×IJ ׳×Ļ +Ġ׾ ×Ļ +Ġ×ŀ ×ķ +Ġng Ãły +Ġn Æ°á»Ľc +Ġ×Ķ ×Ļ×IJ +Ġ×IJ ×Ļ +Ġh Æ¡n +ĠÙĩ Ø°Ùĩ +ĠÙĪ ÙĬ +ĠاÙĦ Ø°ÙĬ +Ġ×ķ ×ŀ +Ġgi á +Ġnh ân +Ġch ÃŃnh +Ġm ình +ĠÐĿ а +Ġth ế +Ġ×Ļ ×ķתר +Ġ×IJ ×Ŀ +Ġn ên +Ġh ợ +Ġhợ p +Ġc òn +ĠÙĩ ÙĪ +Ġc Æ¡ +Ġr ất +ĠVi á»ĩt +Ġب عد +Ġש ×Ļ +Ġth á»Ŀi +Ġc ách +ĠÄij á»ĵng +Ġн о +Ġtr Æ°á»Ŀng +Ø Ł +ĠÄij á»ĭnh +ĠÄiji á»ģu +×Ļ ×Ļ×Ŀ +Ġth á»±c +n ın +Ġh ình +Ġn ói +Ġc ùng +Ġ×Ķ ×Ķ +ĠØ¥ ÙĨ +Ġ×IJ ×ij׾ +Ġnh Æ°ng +Ġbi ết +Ġж е +Ġch úng +ĠÄij ang +ĠØ° ÙĦÙĥ +Ġl ên +Ġkh ách +Ġn Ãło +Ġs á»Ń +Ġkh ác +Ġë° ı +Ġl ý +×Ļ ×Ļ +ĠÄij ây +Ġ׾ ×ŀ +Ġc ần +Ġtr ình +Ġph át +ãģ« ãĤĤ +п о +Ġn Äĥng +Ġb á»Ļ +Ġv ụ +ĠÄij á»Ļ +Ñĩ е +Ġnh áºŃn +Ġtr Æ°á»Ľc +Ġ×¢ ×ĵ +Ġh Ãłnh +ĠØ® ÙĦاÙĦ +Ġl ượng +Ġc ấp +Ġtá» ± +Ġv ì +Ġt Æ° +Ġch ất +Ġ׼ ×ŀ×ķ +Ġg ì +Ġש ׳ +Ġt ế +ת ×ķ +Ġnghi á»ĩp +Ġm ặt +ĠÙĥ Ùħا +Ġ×ij ×Ļף +Ġר ק +Ġth ấy +Ġmá y +ĠÙģ Ùī +Ġd ân +Ġ×IJ ×Ĺ×ĵ +Ġt âm +Ġ׼ ×ļ +Ġ׾ ×ķ +в о +Ġt ác +Ġto Ãłn +ĠÙĪ Ùħ +Ġk ết +Ġ หรืà¸Ń +ĠÙĪاÙĦ Ùħ +ĠÄiji á»ĥm +Ġ×ĸ ×ķ +Ġ×ij ×ķ +׼ ×ķת +Ġh á»Ļi +Ġb ằng +ت Ùĩا +Ġ׼ ×ĵ×Ļ +Ġ×Ķ ×Ŀ +Ġxu ất +ĠÙĤ د +Ġb ảo +Ġt á»ijt +Ġt ình +ĠÙĩ ÙĬ +ĠÄij á»iji +Ġthi ết +Ġhi á»ĩu +Ġti ếp +Ġt ạo +ת ×Ķ +Ġch ủ +o ÅĽÄĩ +Ġgi ú +Ġgiú p +Ġà ½ +Ġqu ả +Ġlo ại +Ġc ô +Ġà ´ +Ġô ng +Ġ×Ķ ×ķ +ĠاÙĦÙĬ ÙĪÙħ +ĠtÃŃ nh +г а +Ġph òng +Ġ Äĥn +Ġع اÙħ +Ġv á»ĭ +lar ını +r ÃŃa +Ġt Ỽi +ĠÄij Æ°á»Ŀng +Ġgi Ỽi +Ġb ản +Ġc ầu +Ġnhi ên +Ġb á»ĩnh +Ġth Æ°á»Ŀng +Ġ×IJ ×Ļף +ĠÄij á»ģ +Ġh á»ĩ +Ġ×Ļש ר×IJ׾ +Ġqu á +ĠÐĹ Ð° +ãģ® ãģ§ãģĻãģĮ +ĠÐŁ ÑĢи +Ġph ần +ĠÙĪ ÙĦا +ĠlỼ n +Ġtr á»ĭ +Ġcả m +Ġм о +Ġd ùng +ĠاÙĦ Ùī +ĠعÙĦÙĬ Ùĩ +ĠìŀĪ ìĬµëĭĪëĭ¤ +ÙĬ ÙĤ +ĠÙĤ بÙĦ +Ġho ặc +ĠØŃ ÙĬØ« +Ġ à¸Ĺีà¹Ī +Ġغ ÙĬر +ĠÄij ại +Ġsá»ij ng +нÑĭ ми +Ġth ức +Ġפ ×Ļ +ĠÄiji á»ĩn +ãģª ãģĭãģ£ãģŁ +Ġgi ải +Ġv ẫn +Ġи Ñħ +Ġö nce +Ġv áºŃy +Ġmu á»ijn +Ġ ảnh +à¹ĥà¸Ļ à¸ģาร +ĠQu á»ijc +Ġk ế +׳ ×IJ +Ġס ×Ļ +Ġy êu +ãģ® ãģĭ +ĠÄij ẹ +ĠÄijẹ p +Ġch ức +Ġy ıl +ĠTür kiye +d é +ĠÙĤ اÙĦ +Ġd á»ĭch +ĠolduÄŁ u +Ġch á»įn +Ġت Ùħ +หà¸Ļ ึà¹Īà¸ĩ +ãģķãĤĮ ãģŁ +Ġph áp +ìĽ Ķ +Ġti á»ģn +ãģĹ ãģ¾ãģĹãģŁ +Ġש ׾×IJ +ÙĦ Ø© +Ġ׾פ ׳×Ļ +Ġ×ij ×Ļת +ĠH Ãł +ĠØŃ Øª +ĠØŃت Ùī +Ġ×¢ ×ķ×ĵ +Ġn ó +Ġth áng +à¹Ģลืà¸Ń à¸ģ +ר ×Ķ +Ġt Äĥng +Ġcá i +Ġtri á»ĥn +Ġ×IJ×ķת ×ķ +ìłģ ìĿ¸ +ĠC ông +Ġ׾×Ķ ×Ļ×ķת +Ġг ода +и Ñİ +Ġب عض +Ġ à¸ģาร +èī¯ ãģĦ +ÙĪ ت +Ġli ên +ĠÐĿ о +ĠÐĿ е +çļĦ ãģª +ĠÙħ ت +ĠÑĤак же +ĠкоÑĤоÑĢ Ñĭе +Ġ×Ļ ×ĵ×Ļ +Ġtr á»įng +ãĤµ ãĤ¤ãĥĪ +ìłģ ìľ¼ë¡ľ +Ġt áºŃp +Ġש ׾×Ļ +íķĺ ê²Į +Ġt Ãłi +ĠÐ ¯ +Ġr á»ĵi +ا Ùĥ +Ġth Æ°Æ¡ng +Ġ×Ķ ×ĸ×Ķ +ĠÙĪ ÙħÙĨ +à¸Ĺีà¹Ī มี +Ġcu á»Ļc +Ġbü yük +ãģ¨ ãģĭ +Ġ×ij ×Ļ×ķתר +Ġl ần +Ġgö re +Ġtr ợ +Ġ×ĺ ×ķ×ij +ÑĤÑĮ ÑģÑı +Ġth á»ijng +Ġ׼ ש +Ġti êu +Ġ×ŀ×IJ ×ķ×ĵ +Ø Ľ +k Äħ +Ġ à¹ĥà¸Ļ +Ġv ấn +Ġש ׾×ķ +ĠÄij á»ģu +Ùģ ت +Ġê²ĥ ìĿ´ +Ġh óa +ĠاÙĦع اÙħ +ĠÙĬ ÙĪÙħ +к ой +Ġbi á»ĩt +ÑģÑĤ о +Ġ×Ķ ×Ļ×ķ +à¸Ĺีà¹Ī à¸Īะ +Ġ×ĵ ×Ļ +Ġ×IJ ×ļ +Ġá n +ص ÙĪر +Ġtr ÃŃ +ĠÐŁÑĢ о +Ġl á»±c +ãģĹãģ¦ ãģĦãģ¾ãģĻ +Ġb Ãłi +Ġ×ĸ ×IJת +Ġb áo +à¸ļ à¸Ļ +ĠëĮĢ íķľ +Ġti ế +Ġtiế ng +Ġb ên +ãģķãĤĮ ãĤĭ +s ión +Ġt ìm +×¢ ×ķ +m é +ни Ñı +ãģ» ãģ© +Ġà¹Ģà¸ŀ ราะ +ب Ø© +Ġë¶ Ħ +Ġ×IJ ×ĸ +à¸Ĺ à¹Īาà¸Ļ +ת ×Ŀ +Ġth êm +Ġho ạt +y ı +×ĸ ×ķ +Ġgi á»Ŀ +Ġb án +à¸Ĥ าย +Ñĩ а +Ġ à¹Ĩ +ĠاÙĦÙħ ت +ĠоÑĩ енÑĮ +Ġb ất +Ġtr ẻ +ÑĤ ÑĢ +ĠØ£ ÙĨÙĩ +ĠØ« Ùħ +Ġ׼ ×ŀ×Ķ +Ġkh ó +Ġr ằng +ĠÙĪ ÙģÙĬ +ни й +Ġho Ãłn +t ó +Ġ×IJ שר +ĠìĥĿ ê°ģ +Ñģ а +Ġ׼ ×ijר +ĠÑįÑĤ ом +lar ının +Ġch Æ°a +з и +Ġd ẫn +ĠÐļ ак +ج ÙĪ +ĠбÑĭ ло +ĠÙĬ ت +n ı +ÅĤ am +ĠÙĪÙĩ ÙĪ +×ij ×ķ +п и +ר ת +Ġqu á»ijc +ж д +ĠÄij Æ¡n +Ùĥت ب +Ġm ắt +ระ à¸ļ +ระà¸ļ à¸ļ +ĠÙĥ اÙĨت +Ġth ân +สิà¸Ļ à¸Ħà¹īา +×Ĵ ×Ļ +Ġph Æ°Æ¡ng +à¹Ħมà¹Ī à¹Ħà¸Ķà¹ī +ĠìĦ ± +ĠC ác +Ġ×Ķ×ŀ ×ķ +ĠÑĤ ем +Ġ×ĵ ×ķ +à¸Ńะ à¹Ħร +Ġv Äĥn +ãģª ãģ®ãģ§ +ĠN á»Ļi +Ġ×¢ ×ķ +ãĤīãĤĮ ãĤĭ +Ġs áng +Ġgö ster +ãģĵãģ¨ ãĤĴ +Ġtaraf ından +Ġм а +ĠпоÑģл е +Ġ׳ ×Ļת +Ġ׳×Ļת ף +Ġл еÑĤ +Ġ׾ ׳×ķ +Ñģ Ñģ +Ġ×Ļ ×ķ +п е +ĠÙĪ ÙĦÙĥ +ĠÙĪÙĦÙĥ ÙĨ +Ġngo Ãłi +ĠÄij á»ĭa +r zÄħd +dz iaÅĤ +ĠÙħ ر +иÑĤÑĮ ÑģÑı +Ġ×IJ×Ĺר ×Ļ +Ġ׾ ׼׾ +à¸Ĥ à¹īà¸Ńม +à¸Ĥà¹īà¸Ńม ูล +Ġб ол +Ġбол ее +جÙħ ع +л еÑĤ +Ġl á»ĭch +ĠÙħ Ø«ÙĦ +Ġ그리 ê³ł +Ġth ứ +ĠdeÄŁ il +ÙĪ ØŃ +Ġש׾ ×ļ +ĠÙħ ØŃÙħد +Ġn ếu +ĠÄij á»ķi +Ġv ừa +Ġm á»įi +Ġо ни +Ġl úc +ĠÙĬ ÙĥÙĪÙĨ +ì§ Ī +Ġש׾ ׳×ķ +ĠÐĶ о +Ġש ׳×Ļ +ล ิ +×IJ פשר +Ġs ức +ê¶ Į +Ġ ứng +à¹Ħมà¹Ī มี +Ø·ÙĦ ب +ĠÑĩ ем +Ġch uyên +Ġth ÃŃch +Ġ×ķ ×Ļ +íķ © +ĠÙħ صر +д о +ĠÄij ất +Ġch ế +à¸Ĭ ืà¹Īà¸Ń +Ġìĭ ł +ĠØ¥ ذا +Ġر ئÙĬس +Ġש ×Ļש +Ġgiả m +Ñģ ка +lar ında +Ġs ợ +ĠtÃŃ ch +ĠÙĦ ÙĥÙĨ +Ġب Ùħ +×¢ ×ķ×ij +×¢×ķ×ij ×ĵ +ÅĤÄħ cz +ları na +Ġש ×Ŀ +ĠÙĦ ت +Ġש×Ķ ×ķ×IJ +t ów +Ġëĭ¤ 른 +ĠØ£ Ùĥثر +ãģ® ãģ§ãģĻ +׼ ×Ļ×Ŀ +ĠolduÄŁ unu +ãģĭ ãģª +ãĤĤ ãģĨ +ÙĬ ØŃ +Ġnh ìn +Ġngh á»ĩ +ãģ«ãģª ãģ£ãģ¦ +п а +Ġquy ết +ÙĦ ÙĤ +t á +Ġlu ôn +ĠÄij ặc +Ġ×IJ ר +Ġtu á»ķi +s ão +ìĻ ¸ +ر د +ĠبÙĩ ا +Ġ×Ķ×Ļ ×ķ×Ŀ +×ķ ×ķ×Ļ +ãģ§ãģĻ ãģŃ +ĠÑĤ ого +Ġth ủ +ãģĹãģŁ ãģĦ +ر ÙĤ +Ġb ắt +г Ñĥ +Ġtá» Ń +ÑĪ а +Ġ à¸Ľà¸µ +Ġ×Ķ×IJ ×Ŀ +íı ¬ +ż a +Ġ×IJת ×Ķ +Ġn á»Ļi +Ġph ÃŃ +ĠÅŁek ilde +Ġl á»Ŀi +d ıģı +Ġ׼×IJ ף +Ġt üm +Ġm ạnh +ĠM ỹ +ãģĿ ãĤĵãģª +Ġnh á»ı +ãģª ãģĮãĤī +Ġb ình +ı p +à¸ŀ า +ĠÄij ánh +ĠÙĪ ÙĦ +ר ×ķת +Ġ×IJ ×Ļ×ļ +Ġch uyá»ĥn +Ùĥ ا +ãĤĮ ãĤĭ +à¹ģม à¹Ī +ãĤĪ ãģı +ĠÙĪ ÙĤد +íĸ Īëĭ¤ +Ġn Æ¡i +ãģ«ãĤĪ ãģ£ãģ¦ +Ġvi ết +Ġà¹Ģà¸ŀ ืà¹Īà¸Ń +ëIJĺ ëĬĶ +اد ÙĬ +ĠÙģ Ø¥ÙĨ +ì¦ Ŀ +ĠÄij ặt +Ġh Æ°á»Ľng +Ġx ã +Ġönem li +ãģł ãģ¨ +Ġm ẹ +Ġ×ij ×Ļ +Ġ×ĵ ×ijר +Ġv áºŃt +ĠÄij ạo +Ġdá»± ng +ĠÑĤ ом +ĠÙģÙĬ Ùĩا +Ġج ÙħÙĬع +Ġthu áºŃt +st ÄĻp +Ġti ết +Ø´ ÙĬ +Ġе Ñīе +ãģĻãĤĭ ãģ¨ +ĠmÃł u +ĠÑįÑĤ ого +Ġv ô +ĠÐŃ ÑĤо +Ġth áºŃt +Ġn ữa +Ġbi ến +Ġn ữ +Ġ׾ ׼×Ŀ +×Ļ ×Ļף +Ġس ت +ĠÐŀ ÑĤ +Ġph ụ +ê¹Į ì§Ģ +Ġ׾ ×ļ +Ġk ỳ +à¹ĥ à¸Ħร +Ġg ây +ĠÙĦ ÙĦÙħ +Ġtụ c +ت ÙĬÙĨ +Ġtr ợ +Ġ׾ פ×Ļ +Ġb á»ij +ĠÐļ а +ĠÄij ình +ow Äħ +s ında +Ġkhi ến +s ız +Ġк огда +ס ׾ +ĠбÑĭ л +à¸Ļ à¹īà¸Ńย +обÑĢаР· +Ġê²ĥ ìĿ´ëĭ¤ +ëĵ¤ ìĿĢ +ãģ¸ ãģ® +Ġà¹Ģม ืà¹Īà¸Ń +Ġph ục +Ġ׊׾ק +Ġh ết +ĠÄij a +à¹Ģà¸Ķà¹ĩ à¸ģ +íĺ ķ +l ÃŃ +ê¸ ī +Ġع دد +ĠÄij á»ĵ +Ġg ần +Ġ×Ļ ×ķ×Ŀ +Ġs Ä© +ÑĢ Ñıд +Ġquy á»ģn +Ġ×IJ ׾×IJ +Ùĩ Ùħا +׳ ×Ļ×Ķ +׾ ×ķת +Ġ×Ķר ×ij×Ķ +Ġti ên +Ġal ın +Ġd á»ħ +人 ãģĮ +но Ñģ +л ÑģÑı +ĠÄij Æ°a +ส าว +иÑĢов ан +Ġ×ŀס פר +×Ĵ ף +Ġki ến +ĠÐ ¨ +p é +б Ñĥ +ов ой +б а +ĠØ¥ ÙĦا +×IJ ׾×Ļ +Ġx ây +Ġb ợi +Ġש ×ķ +人 ãģ® +ק ×Ļ×Ŀ +à¹Ģà¸Ķ ืà¸Ńà¸Ļ +Ġkh á +Ġ×ķ ׾×Ķ +×ĵ ×ķת +Ġ×¢ ×ij×ķר +Ġبش ÙĥÙĦ +ĠÙĩÙĨا Ùĥ +ÑĤ ÑĢа +Ġ íķĺëĬĶ +ร à¸Ńà¸ļ +owa ÅĤ +h é +Ġdi á»ħn +Ġ×Ķ ׼׾ +ĠØ£ س +Ġch uyá»ĩn +ระ à¸Ķัà¸ļ +ĠNh ững +Ġ×IJ ×Ĺת +ĠØŃ ÙĪÙĦ +л ов +׳ ר +Ġ×ķ ׳ +Ġch Æ¡i +Ġiç inde +ÑģÑĤв Ñĥ +Ġph á»ij +ĠÑģ Ñĥ +ç§ģ ãģ¯ +Ġch ứng +Ġv á»±c +à¹ģ à¸Ń +Ġl áºŃp +Ġtừ ng +å°ij ãģĹ +ĠNg uy +ĠNguy á»ħn +ĠÙģÙĬ Ùĩ +Ġб а +×Ļ ×Ļת +Ġ×ľ×¢ ש×ķת +Ġ×ŀ ׼ +Ġnghi á»ĩm +Ġм ного +Ġе е +ëIJĺ ìĸ´ +Ġl ợi +Ġ׾ ׾×IJ +Ġ׼ ף +Ġch ÃŃ +ãģ§ ãģ® +×Ĺ ×ķ +ש ×ķ×Ŀ +Ġ×ŀ ר +ĠÐĶ лÑı +Å ģ +Ġ׼×IJ שר +ĠM á»Ļt +ĠÙĪاÙĦ ت +ĠìĿ´ 룰 +ÅŁ a +Ġchi ến +Ġaras ında +Ġ×ij ×IJתר +ãģķãĤĮ ãģ¦ãģĦãĤĭ +Ø´ ÙĥÙĦ +Ġt ượng +Ġت ت +ĠC ó +Ġb á»ı +Ġtá»ī nh +Ġkh ÃŃ +ĠпÑĢ оÑģÑĤ +ĠпÑĢоÑģÑĤ о +ĠÙĪ ÙĤاÙĦ +Ġgi áo +ĠN ếu +×IJ ×ŀר +×¢×ł×Ļ ×Ļף +íİ ¸ +Ùĩد Ùģ +ĠB á»Ļ +Ġb Ãłn +Ġng uyên +Ġgü zel +ส าย +ì² ľ +×ŀ ×ķר +Ġph ân +ס פק +ק ×ij׾ +ĠاÙĦÙħ تØŃ +ĠاÙĦÙħتØŃ Ø¯Ø© +ائ د +Ġ×IJ ×ŀר +Ġki ÅŁi +ì¤ Ģ +Ġtr uyá»ģn +ĠÙĦ Ùĩا +ĠÐľ а +à¸ļริ ษ +à¸ļริษ ั +à¸ļริษั à¸Ĺ +Ġש ׳×Ļ×Ŀ +Ġмен Ñı +ÅŁ e +Ġdi á»ĩn +Ġ×IJ׳ ×Ĺ׳×ķ +k ü +Ġc á»ķ +Ġm á»Ĺi +w ä +Ùħ ÙĬ +Ġhi á»ĥu +ëĭ ¬ +Ġ×Ķ ×Ĺ׾ +Ġt ên +Ġki á»ĩn +ÙĨ ÙĤÙĦ +Ġv á»ĩ +×ĵ ת +ĠÐłÐ¾ÑģÑģ ии +л Ñĥ +ĠاÙĦع ربÙĬØ© +ĠØ· رÙĬÙĤ +Ġ×Ķ×ij ×Ļת +Ñģ еÑĢ +Ġм не +ä u +Ġtri á»ĩu +ĠÄij ủ +Ġר ×ij +ت ÙĩÙħ +à¸ĭ ี +Ġì§Ģ ê¸Ī +li ÅĽmy +د عÙħ +ãģł ãĤįãģĨ +Ñģки е +Ġh á»ıi +Ġק ×ķ +ÑĢÑĥ Ñģ +ÙĨ ظر +ãģ® ãĤĤ +Ġ×Ķ ׼×Ļ +ĠìĽ IJ +ÙĪ Ùĩ +ĠÙĪ Ùİ +ĠB ạn +п лаÑĤ +Ġ×ŀ ×ŀש +лÑİ Ð± +ĠнÑĥж но +Ġth Æ° +ãģ µ +ãģı ãĤīãģĦ +ر Ø´ +ר ×ķ×Ĺ +ĠÙĬ تÙħ +Ġצר ×Ļ×ļ +Ġph á +ม à¸Ńà¸ĩ +Ġ×ij×IJ ×ķפף +Ġcả nh +Ġíķľ ëĭ¤ +Ġ×Ķ×ŀ ת +à¸ķà¹Īาà¸ĩ à¹Ĩ +มี à¸ģาร +Ñģки Ñħ +ĠÐĴ Ñģе +Ġا ÙĪ +ج ÙĬ +ãģĵãģ¨ ãģ¯ +Ġd Ãłi +Ġh á»ĵ +èĩªåĪĨ ãģ® +à¹Ħ หà¸Ļ +ëĵ¤ ìĿĦ +ĠV Äĥn +Ġд аж +Ġдаж е +Ñĭ ми +лаÑģ ÑĮ +ÙĬ ÙĪÙĨ +ÙĨ ÙĪ +c ó +ãģĹãģ¦ ãģĦãģŁ +ãģł ãģĭãĤī +طاÙĦ ب +Ġc á»Ńa +п ÑĢоÑģ +ãģªãģ© ãģ® +รุ à¹Īà¸Ļ +Ġchi ếc +л Ñĭ +ĠÑıвлÑı еÑĤÑģÑı +Ġn á»ķi +ãģ® ãģĬ +Ġ×IJת ×Ŀ +ĠëķĮ문 ìĹIJ +à¸ģล าà¸ĩ +ĠbaÅŁ ka +ìĦ Ŀ +ĠÑĨ ел +Ùģ ÙĤ +ãģ«ãĤĪ ãĤĭ +ÙĤ ا +Ġçı kar +Ġcứ u +Ø· ا +Ġש ת +à¹Ĥ à¸Ħ +Ġ×ŀ ׾ +Ġ×Ķ פר +Ġг де +ĠØ® Ø· +åīį ãģ« +c jÄĻ +Ġ׊ש×ķ×ij +ר×Ĵ ×¢ +Ġkho ảng +ĠÄij á»Ŀi +ĠÐł е +Ġо на +Ġ×IJ ׳×ķ +ãģ® ãģ« +ĠاÙĦØ° ÙĬÙĨ +кÑĥ п +ãĤµ ãĥ¼ãĥ +ãĤµãĥ¼ãĥ ĵ +ãĤµãĥ¼ãĥĵ ãĤ¹ +в ал +г е +Ġgi ữa +ĠKh ông +ĠâĹ ĭ +à¸ģล ุà¹Īม +ĠÙħÙĨ Ø° +à¸Ń à¹Īาà¸Ļ +ĠÑģп оÑģоб +ĠÄij á»Ļi +Ġdi ÄŁer +Ġ à¸ĸà¹īา +Ùħ Ø«ÙĦ +Ġ×Ķ×IJ ×Ļ +Ġد ÙĪÙĨ +ÙĬر اÙĨ +Ñī и +بÙĨ اء +ĠØ¢ خر +ظ Ùĩر +Ġ×ij ׼ +ĠاÙĦÙħ ع +ãĥ Ĵ +Ġt ất +Ġm ục +ĠdoÄŁ ru +ãģŁ ãĤī +Ġס ×ķ +Ġx ác +ร à¸Ń +ĠcÄĥ n +Ġон л +Ġонл айн +Ġk ý +Ġch ân +Ġ à¹Ħมà¹Ī +اØŃ Ø© +r án +׳×Ļ ×Ļ×Ŀ +Ġ×ij ף +ĠÐ ĸ +à¸ķร à¸ĩ +д Ñĭ +Ġs ắc +ÙĦ ت +ãĥŃ ãĥ¼ +ĠÙĦ ÙĨ +Ġר ×ķ +Ġd Æ°á»Ľi +à¹Ģ à¸ĺ +à¹Ģà¸ĺ à¸Ń +e ÄŁi +Ġ×ķ ש +ĠÙĦ Ø£ +Ġg ặp +Ġc á»ij +ãģ¨ ãģ¦ãĤĤ +رÙĪ س +Ġ׾×Ķ ×Ļ +Ġë³ ¸ +ä¸Ĭ ãģĴ +Ġm ức +Ñħ а +Ġìŀ ¬ +à¸ī ัà¸Ļ +ÑĢÑĥ ж +Ġaç ık +ÙĪ اÙĦ +Ġ×ĸ ×ŀף +人 ãģ¯ +ع ÙĬÙĨ +Ñı Ñħ +Ġ×Ĵ×ĵ ×ķ׾ +ר ×ķ×ij +g ó +ëĿ¼ ê³ł +Ġark adaÅŁ +ÙĨ شر +Ġгод Ñĥ +ĠболÑĮ ÑĪе +ãģ¡ãĤĩ ãģ£ãģ¨ +Ġcâ u +Ġs át +íĶ ¼ +Ġti ến +íķ´ ìķ¼ +ĠÙĪ Ø£ÙĨ +à¸Ļ าà¸Ļ +Ġ×ij×IJ×ŀ צע +Ġ×ij×IJ×ŀצע ×ķת +Ġ׾ ר +Ġqu ản +ĠÙĪاÙĦ Ø£ +Ġ×IJ×ķת ×Ķ +Ġìĸ´ëĸ ¤ +Ġê²ĥ ìĿĢ +ØŃس ÙĨ +Ġm ất +à¸Ħ ูà¹Ī +ãĥ¬ ãĥ¼ +ĠÐĶ а +Ġol ması +Ġthu á»Ļc +׳ ×Ĺ +íĨ ł +Ġsö yle +ãģĿãģĨ ãģ§ãģĻ +Ġت ÙĥÙĪÙĨ +л ÑĥÑĩ +׾ ×Ļ×ļ +ĠØ£ ØŃد +ли ÑģÑĮ +ĠвÑģ его +Ġ×Ķר ×ij +Ġëª » +o ÄŁ +oÄŁ lu +ĠìĦ ł +Ġк аÑĢ +à¸łà¸² à¸Ħ +e ÅĦ +Ġ à¸ģà¹ĩ +Ġa ynı +Ġb Ãł +ãģªãĤĵ ãģ¦ +Ġ모 ëĵł +ÙĤر ار +ãģĹãģª ãģĦ +ĠÐĴ о +ĠÙĪÙĩ ÙĬ +ни ки +ãĤĮ ãģŁ +Ġchu ẩn +ר ×¢ +Ùģ رÙĬÙĤ +ãĤĴ åıĹãģij +ĠÄij úng +б е +׼ ×ķ×Ĺ +п Ñĥ +Ġ×ķ ×Ĵ×Ŀ +×ŀ ׳×Ļ +íĸ ¥ +צ ×Ļ×Ŀ +à¸ĭ ิ +Ùĩ ÙĨ +н ем +Ġ×ij×ij ×Ļת +ر ع +Ġ ส +ĠÄIJ Ãł +íķĺ ëĭ¤ +Ġ ấy +×Ĺ ×ķ×ĵ +×Ĺ×ķ×ĵ ש +ĠÑĩеÑĢ ез +Ñĥ л +ĠB ình +Ġê²ĥ ìĿĦ +Ġ×Ĵ ר +ä»ĺ ãģij +×Ĺ׾ ק +Ġت ÙĦÙĥ +à¹ĥส à¹Ī +sz Äħ +ÙĤ اÙħ +د ÙĪر +ĠÙģ ÙĤØ· +Ġh ữu +Ġмог ÑĥÑĤ +Ġg á»įi +Ġק ר +à¸Īะ มี +ت ÙĤدÙħ +Ġع بر +Ġ׾×Ķ ×Ŀ +ĠÑģам о +ס ×ĵר +Ġc Ãłng +r ÃŃ +Ġìŀ ¥ +ëĵ¤ ìĿĺ +ĠÙĦ Ùĥ +п оÑĢÑĤ +Ġkh ả +ĠÑģеб Ñı +׳ ף +Ġد ÙĪر +Ġm ợ +Ġcâ y +Ġf ark +Ġfark lı +а ÑİÑĤ +Ġtr á»±c +wiÄĻks z +Ġthu á»ijc +Ġت ØŃت +ت ÙĦ +ов Ñĭе +ëĤ ł +Ġв ам +بÙĦ غ +Ġê°Ļ ìĿĢ +íĮ IJ +ÙĦ ب +Ġnas ıl +Ġод ин +м ан +ĠعÙĦÙĬ Ùĩا +б и +Ġפ ש×ķ×ĺ +×ijר ×Ļ +Ġש ׳×Ķ +Ġëı Ħ +ĠÄIJ ại +Ġ×IJ×ķת ×Ŀ +ĠاÙĦØŃ Ø± +Ġб о +à¸Ī ุà¸Ķ +Ġr õ +ĠdeÄŁi ÅŁ +Ġëĭ ¨ +ĠÑģлÑĥÑĩ а +ĠÑģлÑĥÑĩа е +Ġ×IJ׳ ש×Ļ×Ŀ +×ĵ ×£ +ש×ij ת +Ġש׾ ׼×Ŀ +Ġch ú +nik ów +Ġtan ı +Ġcá o +ĠÄij á +Ġ×IJ ×ĵ×Ŀ +Ġê° ķ +Ġnhi á»ĩm +Ġ׾ ס +Ġ×Ľ×ª ×ij +Ġ×Ķס פר +ĠÄij Äĥng +Ġë ijIJ +à¸ľ ิ +à¸ľà¸´ ว +ج ا +Ġê° IJ +ر Ø£ +ست خدÙħ +ãģ«ãģªãĤĬ ãģ¾ãģĻ +Ġtá» · +×ĺ ×ķר +г овоÑĢ +Ġв оÑģ +ĠÙħÙĨ Ùĩا +иÑĢов аÑĤÑĮ +ĠÄij ầy +׳ ×Ĵ +ĠÙħ ÙĪ +ĠÙħ ÙĪÙĤع +ר׼ ×Ļ +ت Ùı +ëª ¨ +Ġת ×ķ +ÙĬا Ùĭ +à¹ĥ à¸Ķ +ãĤĬ ãģ¾ãģĻ +à¸Ńยูà¹Ī à¹ĥà¸Ļ +ĠØ£ ÙĪÙĦ +ĠØ£ خرÙī +Ġc Æ° +ص ار +×ŀ׊ש×ij +б ÑĢа +ÅĦ ski +б ÑĢ +ĠÙĬ Ùı +à¸ģ ิà¸Ļ +Ġch á»ijng +Ùħ Ùı +Ġ à¸Ħืà¸Ń +Ġت ÙĨ +t ÃŃ +y Äĩ +Ġm ạng +Ùģ ÙĪ +Ġdü nya +ק ר×IJ +Ġק ׾ +ĠØŃ Ø§ÙĦ +c ÃŃa +Ġà¹Ģ รา +Ġר ×ķצ×Ķ +Ġá p +ë° ķ +ا ÙĤØ© +ни Ñİ +Ġ×IJ ׾×ķ +Ġ×ŀס ×ķ +ãģ§ãģ¯ ãģªãģı +Ġtr ả +Ġק שר +mi ÅŁtir +Ġl Æ°u +Ġh á»Ĺ +ĠбÑĭ ли +Ġl ấy +عÙĦ Ùħ +Ġö zel +æ°Ĺ ãģĮ +Ġ×ĵ ר×ļ +Ùħ د +s ını +׳ ×ķש×IJ +r ów +Ñĩ еÑĢ +êµIJ ìľ¡ +ĠÐľ о +л ег +ĠV Ỽi +วัà¸Ļ à¸Ļีà¹ī +ÑİÑī ие +ãģĬ ãģĻ +ãģĬãģĻ ãģĻ +ãģĬãģĻãģĻ ãĤģ +ëı ħ +Ġ×Ļ×Ķ ×Ļ×Ķ +×ŀ ×ĺר +Ñı ми +Ġl á»±a +ĠÄij ấu +à¹Ģส ียà¸ĩ +Ġt Æ°Æ¡ng +ëĵ ± +ĠÑģÑĤ аÑĢ +à¹ĥ à¸ļ +ว ัà¸Ķ +ĠÄ° stanbul +Ġ à¸Īะ +à¸ķ ลาà¸Ķ +Ġب ÙĬ +à¹ģà¸Ļ ะ +à¹ģà¸Ļะ à¸Ļำ +س اعد +Ġب Ø£ +Ġki á»ĥm +ØŃ Ø³Ø¨ +à¸Ĭั à¹īà¸Ļ +Ġ×ķ ×¢×ķ×ĵ +ов ÑĭÑħ +оÑģ нов +Ġtr Æ°á»Łng +צ ×ij×¢ +ĠÃŃ t +Ġk ỹ +cr é +Ñı м +êµ ° +ãģĮ ãģªãģĦ +ÙĬÙĦ Ø© +ãĥķ ãĤ£ +ر Ùī +ĠÙĬ جب +Ġ×IJ ×£ +Ġc á»±c +ãĤīãĤĮ ãģŁ +Ġ à¸ľà¸¹à¹ī +Ġ à¸Ń +lar ımız +Ġkad ın +Ġê·¸ ëŀĺ +Ġê·¸ëŀĺ ìĦľ +ĠëĺIJ ëĬĶ +ĠÄij ả +ĠÄijả m +Ġ×IJ ×ķ×ŀר +Ġy ếu +ci Äħ +ciÄħ g +Ġt á»ij +Ġש×IJ ׳×Ļ +Ġdz iaÅĤa +Ñī а +ĠÄij Ãłn +s ına +ãģĵãĤĮ ãģ¯ +Ġ×ij ׾×Ļ +Ġ×ij ×Ļשר×IJ׾ +л оÑģÑĮ +Ġgi ữ +ê° IJ +ÑĢ он +تج ار +г лав +в ин +Ġh ạn +Ġyapı lan +ب س +Ġ à¸ŀรà¹īà¸Ńม +ê´Ģ 리 +mÄ±ÅŁ tır +b ü +r ück +ĠBaÅŁkan ı +ĠÙĦ ÙĬس +Ġs Æ¡ +à¸Īัà¸ĩ หว +à¸Īัà¸ĩหว ัà¸Ķ +د اء +Ġ×Ķ ׼ +v ÃŃ +ש ×IJר +Ġh Æ°á»Łng +Ġb óng +ĠCh ÃŃnh +Äħ c +à¹Ģà¸ģีà¹Īยว à¸ģัà¸ļ +Ġtá» © +Ġtứ c +ĠÑĨ веÑĤ +Ġt á»iji +ĠnghÄ© a +ÙĦا عب +د ÙĦ +Ġפע ×Ŀ +h ör +à¸Ĭ ุà¸Ķ +à¸ŀ ู +à¸ŀู à¸Ķ +п аÑģ +ĠÅŁ u +Ġt Æ°á»Łng +خار ج +Ġâ m +ĠинÑĤеÑĢ еÑģ +ен нÑĭÑħ +×IJ ׳×Ļ +بد Ø£ +ëĿ¼ ëĬĶ +ì¹ ´ +æĸ¹ ãģĮ +ли в +Ġ à¸Ħà¸Ļ +ער ×ļ +à¸Ĥà¸Ńà¸ĩ à¸Ħุà¸ĵ +п ад +Ġc ạnh +ĠëĤ ¨ +ĠÄij âu +Ġbi á»ĥu +ãĤĤ ãģĤãĤĭ +׾ ×Ĵ +Ġ สำหรัà¸ļ +Ġxu á»ijng +ס ×ķ +ĠØ° ات +ĠÐľ е +ع اÙĦÙħ +×IJ ס +ب ÙĬØ© +Ø´ ا +и ем +ĠNg Æ°á»Ŀi +íĺ ij +Ñģл ов +Ġп а +Ġm ẫu +ĠпÑĢоÑĨ еÑģÑģ +ĠNh Ãł +пÑĢо из +пÑĢоиз вод +à¸łà¸²à¸¢ à¹ĥà¸Ļ +Ġ à¸ļาà¸Ĺ +×ŀ ׳×ķ +ĠоÑĢг ан +רצ ×ķ +×ķ×ŀ ×Ļ×Ŀ +Ġyaz ı +Ġd ù +ãĥ¬ ãĥ³ +ÙĪÙĦ ÙĬ +ย ู +Ġtr ò +à¹Ģà¸ŀ ลà¸ĩ +Ġ×ŀ ׾×IJ +à¸ķ ล +à¸ķล à¸Ńà¸Ķ +ĠÄij ạt +Ġ×Ĺ×ĵ ש +p óÅĤ +Ġ×ŀ ×ĵ×Ļ +ujÄħ c +×ŀ׳×Ķ ׾ +Ġש×ij ×ķ +Ġ×Ķ×ŀש פ×ĺ +Ġ×IJ ׾×Ķ +ĠÙĪ Ø°ÙĦÙĥ +à¹Ģà¸ŀ ราะ +ĠÄijo Ãłn +Ġíķ¨ ê»ĺ +Ġd ục +Ø´ ت +Ġ ula +Ġula ÅŁ +Ġqu ý +Ġ×Ķ ×Ĵ×ĵ×ķ׾ +à¸ķัà¹īà¸ĩ à¹ģà¸ķà¹Ī +Ġש ר +Ø´ Ùĩد +׳ ש×Ļ×Ŀ +à¸ŀ ล +رÙĪ ا +ãĤĮ ãģ¦ +Ġн иÑħ +Ġдел а +ãģ§ãģį ãģªãģĦ +ÅĤo ż +×IJ ×Ĺר +ì ½Ķ +ãĤ¢ ãĥĥãĥĹ +د Ùģع +Ġti á»ĩn +Ġkh á»ı +Ġkhá»ı e +ĠاÙĦع اÙħØ© +ãģ« ãģĤãĤĭ +ĠÄij á»Ļc +ì¡ ± +Ġc ụ +й ÑĤе +Ġзак он +ĠпÑĢо екÑĤ +ìĸ ¸ +ÙĦ ØŃ +ĠçalÄ±ÅŁ ma +ãĤĴ ãģĻãĤĭ +Ñħ и +ع اد +Ġ׳ ×ŀצ×IJ +Ġר ×Ļ +à¸Ńà¸Ńà¸ģ มา +ĠT ôi +Ġth ần +ĠÙĬ ا +ล าย +Ġав ÑĤо +Ġsı ra +ĠÙĥ Ø«ÙĬر +Ùħ ÙĬز +ĠاÙĦع ÙĦÙħ +æĸ¹ ãģ¯ +×ķ×¢ ×ĵ +Ġобла ÑģÑĤи +×Ļ׾ ×Ļ×Ŀ +ãģĮ åĩº +à¸ĺ ุ +à¸ĺุ ร +à¸ĺุร à¸ģิà¸Ī +ÙĤت ÙĦ +ר×IJ ×ķ +Ġng u +Ġngu á»ĵn +Ġ มา +Ġпл ан +t ório +Ġcu á»iji +Ñģк ом +ĠاÙĦÙħ اض +ĠاÙĦÙħاض ÙĬ +Ġ×ij×¢ ׾ +Ġר ×ij×Ļ×Ŀ +Ġlu áºŃn +Ùĥ ÙĪ +à¸Ĺัà¹īà¸ĩ หมà¸Ķ +в ан +Ġtho ại +à¹Ħ à¸Ń +б иÑĢ +ĠاÙĦ ض +ت ا +ĠÑĢ од +ĠV Ãł +×ŀ ×Ļף +ĠбÑĭ ла +к ами +ĠÐĶ е +t ık +קר ×Ļ +ĠeÄŁ itim +ĠÙĥ بÙĬر +ب Ùĥ +ĠÙĦ ÙĪ +в ой +Ġ ãģĵãģ® +ĠÑĤ ÑĢÑĥд +my ÅĽl +Ġs Æ° +à¸ŀ ีà¹Ī +Ġ à¹ģลà¹īว +×¢ ק +Ġ×Ĺ×ijר ת +ระ หว +ระหว à¹Īาà¸ĩ +×Ļ ×Ļ×Ķ +ĠاÙĦÙĨ اس +ün ü +Ġ׾ ×ŀ×Ķ +Ġch Æ°Æ¡ng +ĠH á»ĵ +ار ت +ãĤĪãģĨ ãģ§ãģĻ +l á +ק×Ļ ×Ļ×Ŀ +æľ¬ å½ĵ +æľ¬å½ĵ ãģ« +ãģĵãĤĵ ãģª +Ñģ ов +Ġ×ķ ×Ĺ +à¹Ģà¸ģ à¹ĩà¸ļ +Ġк ÑĤо +à¹Ĥร à¸Ħ +ĠØ´ رÙĥØ© +ع زÙĬ +عزÙĬ ز +Ø·ÙĦ ÙĤ +п ÑĥÑģÑĤ +Ùģ تØŃ +ëŀ Ģ +Ġhã y +ض Ùħ +ë¦ ° +åł´åIJĪ ãģ¯ +ãĤª ãĥ¼ +Ġh ắn +Ġ×IJ ×ij×Ļ×ij +Ġש׾×Ķ ×Ŀ +Ġ×Ķ×Ļ ×Ļת×Ķ +ĠاÙĦد ÙĪÙĦØ© +ĠاÙĦ ÙĪÙĤ +ĠاÙĦÙĪÙĤ ت +ãģĤ ãģ¾ãĤĬ +Ġta ÅŁÄ± +Ä° N +×¢ סק +ãģ¦ ãģĦãģŁ +Ġtá»ķ ng +ĠاÙĦØ¥ ÙĨس +ĠاÙĦØ¥ÙĨس اÙĨ +ÑĢ еÑĪ +Ġg ái +ĠÑĨ ен +ĠÙģ ÙĤد +Ùħ ات +ãģķãĤĵ ãģ® +Ġph ù +×ĺ ×Ķ +ĠÙĪاÙĦ تÙĬ +Ġب Ùĥ +ìĿ´ ëĤĺ +к Ñģ +Ùħ ÙĬر +Ġv ùng +ĠاÙĦØ´ عب +ĠNh Æ°ng +ãĥĢ ãĥ¼ +Ġ×Ĺ×Ļ ×Ļ×Ŀ +ĠØ´ خص +ק ×ķ×ĵ +ê² Ģ +×¢ ש +×¢ ×ķ׾×Ŀ +צ ×ķר +ع ÙĤد +ĠiÅŁ lem +Ġ×Ķ×ij ×IJ +Ġd ưỡng +à¸Ł รี +Ġph ÃŃa +ãģ®ä¸Ń ãģ§ +Ġп и +Ġng Ãłnh +ним а +ĠÙĩ ÙĦ +Ġ×ķ ×IJת +ĠÄij áng +é quipe +ĠÑįÑĤ оÑĤ +Ġgö rev +ë§ ¤ +Ġqu ân +å¼ķ ãģį +æĻĤ ãģ« +Ġب Ùħا +×ŀ ×Ļת +Ġü lke +Ġ×ŀק ×ķ×Ŀ +×ij ף +æ°Ĺ æĮģãģ¡ +Ġë§İ ìĿĢ +Ġyük sek +ÑĨ енÑĤÑĢ +ĠÙħ جÙĦس +ç§ģ ãģ® +ÙĤد ر +Ġë¶Ģ ë¶Ħ +Ġì° ¨ +خر ج +ãģĭ ãģªãĤĬ +ë³´ ëĭ¤ +Ġ×ŀ ×Ļ×ĵ×¢ +peÅĤ ni +Ġx á»Ń +ìĹIJìĦľ ëĬĶ +ĠباÙĦ Ùħ +ĠÙĪ Ùħا +ĠÑįÑĤ ой +ب ÙĬÙĨ +n ü +ØŃ Ø² +ØŃز ب +ĠÑĢабоÑĤ а +ĠNh áºŃt +ÙĦ اء +Ġëĵ ¤ +Ġëĵ¤ ìĸ´ +ãĤĦãģĻ ãģĦ +×Ĺ×ĸ ק +Ġ×Ķ×Ĺ ×ijר×Ķ +п иÑĤ +ãģĭãĤī ãģ® +Ġë§IJ ìĶĢ +Ġפ ×ķ +ÙĦ Ùİ +à¹Ģà¸ķà¹ĩ ม +ĠÐļ о +Ġm ówi +Ġt ÃŃn +ר×Ĵ ש +פר ק +Ġtr ạng +ĠÐŀ н +×Ĺ ×ķ×¥ +ĠعÙĨد Ùħا +Ġب ر +使 ãģĦ +Ġr á»Ļng +ëĮĢ ë¡ľ +íĪ ¬ +Ġktóry ch +в ид +ลูà¸ģ à¸Ħà¹īา +Ġmog Äħ +Ġש ×Ĺ +×ij ×Ĺר +ãĥĸ ãĥŃãĤ° +ĠTh Ãłnh +Ġ×Ķ ר×Ļ +ĠÑģÑĤ аÑĤÑĮ +ĠH á»Ļi +à¸ļ à¹īาà¸ĩ +çī¹ ãģ« +ĠÄIJ ức +èĢħ ãģ® +×¢ ×ŀ×ķ×ĵ +×ĺר ×Ķ +Ð ¥ +ĠÙħ Ùħا +Ġe ÅŁ +ĠнеобÑħодим о +ник ов +Ġüzer inde +a ÅĤa +Ġchá»ĭ u +ĠاÙĦ دÙĬÙĨ +أخ بار +ĠÄij au +ãģĮ å¤ļãģĦ +jÄħ cych +د Ø®ÙĦ +ları nd +larınd an +Ġs ẻ +à¸ŀิ à¹Ģศ +à¸ŀิà¹Ģศ ษ +ת ף +t ıģı +Ġlu áºŃt +ĠÅŀ e +ãĤ« ãĥ¼ +ãģ® ãģĤãĤĭ +Ġ×Ķ×IJ תר +ĠاÙĦØ¢ ÙĨ +ıld ı +Ġá o +ĠнаÑĩ ал +Ġvi á»ĩn +Ġ×ij×¢ ×ķ׾×Ŀ +з наÑĩ +×Ļ×ĺ ×Ķ +к ам +ĠÐĺ з +à¹Ģà¸Ĥ ียà¸Ļ +à¸Ļ à¹īà¸Ńà¸ĩ +ÑĤ ÑĢо +à¹Ģ à¸Ł +Ġжиз ни +Ġ สà¹Īวà¸Ļ +Ġv áºŃn +Ġê´Ģ 볨 +Ġl âu +ס ×ĺר +ק ש +س ÙĬر +Ġ×IJ×ķת ×Ļ +Ġm ôi +ائ ب +Ġо ÑģÑĤа +Ġm ón +Ġ×ij ×ŀק×ķ×Ŀ +Ġد اخÙĦ +Ġ×IJ ×ķר +Ġв аÑģ +Ùĥ Ø´Ùģ +ìĺ ¨ +à¸ĸ à¹Īาย +Ġkullan ıl +Ġt ô +ãģ« ãĤĪãĤĬ +ĠëĺIJ íķľ +Ġ×¢×ij×ķ×ĵ ×Ķ +Ġri ê +Ġriê ng +Ġyak ın +ز ا +Å » +×IJ ×ķ׼׾ +شار Ùĥ +Ġб еÑģ +× ´ +Ġا بÙĨ +ĠTá»ķ ng +ÙĨ ظ +ÅĽwi ad +ãĤµ ãĥ¼ +ห าย +ĠG ün +Ġhakk ında +à¹Ģà¸Ĥà¹īา มา +ز ÙĨ +ĠÐł о +Ġbi á»ĥn +ãģ© ãģĵ +Ùģ عÙĦ +ز ع +פר ×ĺ +Ġ×Ķ ף +Ø£ ÙĩÙĦ +Ġth ất +ØŃ ÙħÙĦ +Ñĩ Ñĥ +ĠìĤ¬ ìĭ¤ +ì° ¸ +ĠìľĦ íķ´ +ÙĪ ظ +ĠÐŁ од +Ġkho ản +ÑĤ ен +ĠÙģ اÙĦ +Ñģ ад +à¸Ļ à¸Ńà¸Ļ +ĠاÙĦسعÙĪد ÙĬØ© +" ØĮ +ĠاÙĦ ÙĴ +ãĤī ãģļ +Ġto án +Ġch ắc +׼ ×Ļר +m éd +méd ia +ز ÙĪ +Ġyan ı +פ ׳×Ļ×Ŀ +ØŃ Ø¸ +Ġб еÑģп +ĠбеÑģп лаÑĤ +ĠбеÑģплаÑĤ но +ĠØ£ ÙħاÙħ +à¸Ń าย +à¸Ńาย ุ +ר שת +Ġg á»ĵ +Ġgá»ĵ m +Ġu á»ijng +ص ب +k ır +ãĥij ãĥ¼ +Ġ׾×ĵ עת +Ġк ÑĥпиÑĤÑĮ +׾ ×ķ×Ĺ +ÙĪض ع +ÙĤÙĬ Ùħ +à¸Ľ า +ж ив +à¸Ķ ิà¸Ļ +×IJ ×ķפ +à¹Ģล à¹ĩà¸ģ +ãĥĥ ãĥī +иÑĩеÑģки Ñħ +ĠCh ủ +кÑĢ аÑģ +ÙĪ صÙĦ +p ÅĤat +м оÑĢ +Ġ×Ķ×IJ ×ķ +à¸Ń ิà¸Ļ +Ġíķľ êµŃ +гÑĢ е +Ġìłľ ê³µ +ì° ½ +Ġê°ľìĿ¸ ìłķë³´ +Ġngh á»ĭ +à¸ĭ า +ØŃس اب +Ġby ÅĤa +ÙħÙĦ Ùĥ +иÑĩеÑģки е +Ġb ác +ض ØŃ +ê¸ ¸ +ש ×ŀ×¢ +Ġìĸ´ëĸ » +Ġìĸ´ëĸ» ê²Į +ìĽ Į +ات Ùĩ +à¹Ĥรà¸ĩ à¹ģ +à¹Ĥรà¸ĩà¹ģ รม +خد ÙħØ© +ĠÐł а +׼×ķ׾ ×Ŀ +×ŀש ×Ĺק +ĠÙĪ ÙĥاÙĨ +ס ×ķ×£ +ĠاÙĦØŃÙĥÙĪÙħ Ø© +Ġ×ij ×ĺ +Ġtr áºŃn +Ġ×Ķ×¢ ×ķ׾×Ŀ +ĠÃŃ ch +t Äħ +ש×ŀ ×ķ +Ġ×Ķר×IJש ×ķף +Ġíķĺ ê³ł +ãģķ ãĤī +ãģķãĤī ãģ« +ãģ« ãģĹãģ¦ +Ġ à¸ľà¸¡ +ãģ® ãĤĪãģĨãģª +ĠÙĪ ÙĤت +ãĥį ãĥĥãĥĪ +ÙĦ عب +ÙĪ Ø´ +ìĺ ¬ +Ġ หาà¸ģ +Ġm iaÅĤ +à¸Ĺ à¸Ńà¸ĩ +иÑĤ а +ا صر +ил ÑģÑı +з е +à¸Ľà¸£à¸° มาà¸ĵ +ãģĿãĤĮ ãģ¯ +Ġb ır +Ġbır ak +صÙĨ اع +Ð ® +Ø´ عر +Ġ׳ ×Ĵ×ĵ +Ġب سبب +ãĥĿ ãĤ¤ +ãĥĿãĤ¤ ãĥ³ãĥĪ +ĠاÙĦج ÙĪ +ĠнеÑģк олÑĮко +Ġki ếm +Ùģ Ùİ +Ġض د +×ij×Ļ×ĺ ×ķ×Ĺ +تاب ع +ÙĨ ز +ĠB ản +Ġaç ıkl +Ġaçıkl ama +Ġ à¸Ħุà¸ĵ +à¸Ĺ า +ÅĤ ów +Ø· ب +ÙĨ ØŃÙĨ +Ġ×ŀק ×ķר +ĠÄ° s +Ġдом а +Ġ วัà¸Ļ +Ġd Ãłnh +Ñı н +ми ÑĢ +Ġm ô +ĠvÃł ng +ص اب +s ının +à¸Ħ ืà¸Ļ +Ø® بر +×ĸ׼ ×ķ +Ġ×ŀ ש×Ķ×ķ +m ü +Ġкомпани и +Ġ×Ķ×¢ ×Ļר +ĠÙĥ ÙĪ +ÙĤÙĦ ب +ĠlỼ p +и ки +׳ ×ij +à¹Ĥ à¸Ħร +à¹Ĥà¸Ħร à¸ĩ +à¹Ĥà¸Ħรà¸ĩ à¸ģาร +×ŀ×ķ×¢ ×ĵ +ÑıÑĤ ÑģÑı +หลัà¸ĩ à¸Īาà¸ģ +ени Ñİ +Ġש ×¢ +Ġb Æ°á»Ľc +ãĥ¡ ãĥ¼ãĥ« +ãĤĦ ãĤĬ +Ġ×Ļ×ķ×ĵ ×¢ +Ġê´Ģ íķľ +ĠاÙĦØ£ Ùħر +Ġböl ge +ĠÑģв ой +ÙĦ س +Ġ×ŀ×Ļ ×ķ×Ĺ×ĵ +ĠëĤ´ ìļ© +ĠØ£ جÙĦ +ĠÄIJ ông +Ġ×ŀ ×ł×ª +Ġìĭľ ê°Ħ +Ùĥ Ùİ +ãģ¨ãģĦãģĨ ãģ®ãģ¯ +Ġnale ży +تÙĨظ ÙĬÙħ +ĠÑģозд а +Ġph é +Ġphé p +ãģ§ãģį ãģ¾ãģĻ +Ġع ÙĦÙħ +大ãģį ãģª +ãĤ² ãĥ¼ãĥł +í ħĮ +Ġ׼×ķ׾ ׾ +ĠинÑĤеÑĢ неÑĤ +ĠT ừ +ãģ¨ ãģªãĤĭ +ز اÙĦ +Ġktóry m +Ġnh é +ìĪ ľ +н ев +д еÑĢ +ãĤ¢ ãĥĹãĥª +i á»ĩu +×ij ×Ļ׾ +Ġت س +ĠÄIJ ây +ĠاÙĦØ® اصة +Ġà¹Ģ à¸Ĭ +Ġà¹Ģà¸Ĭ à¹Īà¸Ļ +ص اد +Ġd ạng +س عر +Ġש ×Ļ×ŀ×ķש +×Ĵ ×Ļ×Ŀ +ãģĮãģĤ ãģ£ãģŁ +п ÑĢов +пÑĢов од +Ġ×IJ ×Ļ׳×ķ +Ġ׾ ר×IJ +Ġ׾ר×IJ ×ķת +ĠØ£ ÙģضÙĦ +ĠØŃ ÙĦ +ĠØ£ بÙĪ +ê° ķ +Ġì§ ij +ãģ® ãĤĪãģĨãģ« +Ġפ ׳×Ļ +ס ×Ļ×Ŀ +ĠÙĪÙĩ ذا +Ġka ç +Ġé én +Ġê± ´ +ë° Ķ +Ñĥ з +à¸Ĥà¸Ńà¸ĩ à¹Ģรา +i ÅĤ +ĠÐľ Ñĭ +Ġch ết +ĠاÙĦØ« اÙĨÙĬ +×IJ ק +Ġ×ķ ×¢×ľ +ĠاÙĦØ· ب +×ij×ĺ ×Ĺ +Ġج دÙĬدة +Ġع دÙħ +ع ز +สิà¹Īà¸ĩ à¸Ĺีà¹Ī +ãģĻ ãĤĮãģ° +ĠÄij ô +ì£ ł +د ÙĤ +н омÑĥ +Ġk á»ĥ +ãĤ¢ ãĥ³ +å¤ļãģı ãģ® +à¸Ľà¸£à¸° à¸ģ +à¸Ľà¸£à¸°à¸ģ à¸Ńà¸ļ +פע×Ļ׾ ×ķת +ĠÑģÑĤ ол +may ı +ãģ¤ ãģĦ +Ġyılı nda +Ġ à¸Īึà¸ĩ +koÅĦ cz +ĠTh ông +Ġак ÑĤив +н ÑģÑĤ +нÑģÑĤ ÑĢÑĥ +ĠÃĸ z +Ġת ×ŀ×Ļ×ĵ +ĠÙĥ ÙĨت +Ñģ иÑģÑĤем +pr és +prés ent +Ġn â +Ġnâ ng +gÅĤ os +ĠÙĪز ÙĬر +ØŃ ØµÙĦ +Ġиме еÑĤ +ØŃ Ø±ÙĥØ© +à¸ŀ à¹Īà¸Ń +ãĤĴ ãģĬ +Ġاست خداÙħ +×IJ×Ļר ×ķ×¢ +ä»ĸ ãģ® +Ġש×Ķ ×Ŀ +ãģĹãģŁ ãĤī +ש×ŀ ×Ļ +Ñģ ла +m ı +Ġbaz ı +Ġíķĺ ì§Ģë§Į +×ĵ ׾ +Ġyapt ıģı +ãĥĬ ãĥ¼ +׾ ×Ļ׾×Ķ +ãģ¨ãģĦ ãģ£ãģŁ +änd ig +ĠÅŁ a +ĠÙģÙĬ Ùħا +иÑĤ елÑı +×ŀ ×ķש +à¸Ĥ à¸Ńà¸ļ +l ük +Ġh á»ĵi +Ġëª ħ +ĠاÙĦÙĥ Ø«ÙĬر +צ ×IJ +Ġhaz ır +طر Ùģ +ا ÙĬا +ĠÄij ôi +ен д +ÙĦ غ +×Ĺ ×ĸ×ķר +ĠвÑģ ег +ĠвÑģег да +ëIJĺ ê³ł +×ĵ ×ķ×ĵ +ан а +د ÙĪÙĦØ© +Ġho ạch +ع ÙĦا +عÙĦا ج +Ġ×ķ ×¢×ĵ +×Ķ ×Ŀ +ки й +ÙĦ ÙIJ +Ġ×¢ ׾×Ļ×ķ +ÑİÑī ий +Ġng ủ +صÙĨ ع +ĠاÙĦع راÙĤ +à¸ķà¹Īà¸Ń à¹Ħà¸Ľ +ãģŁãģı ãģķãĤĵ +Ġph ạm +ÙĦ اÙĨ +ات Ùĩا +Ġbö yle +تÙĨ ÙģÙĬ +تÙĨÙģÙĬ Ø° +Ġש×Ķ ×Ļ×IJ +Ñģ Ñĥ +ย าว +Ġש ×ķ׳×Ļ×Ŀ +Ġ×ŀ ×ķ׾ +ĠÑģ ил +Ġ×IJ×Ĺר ×Ļ×Ŀ +Ġph ủ +ÙĤØ· ع +ĠTh ủ +à¸Ľà¸£à¸°à¹Ģà¸Ĺศ à¹Ħà¸Ĺย +ÙĨ ÙĤ +ĠÄijo ạn +Ġب Ø¥ +п ÑĢедел +×ķת ×ķ +Ġy arı +пÑĢ е +ĠczÄĻ ÅĽci +ØŃ ÙĥÙħ +×ķ׳ ×Ļת +פע ׾ +ãĤĴ ãģĹãģ¦ +Ġktó rzy +׾ ×Ŀ +ĠÄIJi á»ģu +ĠкоÑĤоÑĢ аÑı +ĠìĿ´ ìĥģ +ãģĤ ãģ£ãģŁ +Ġ×ŀ×ĵ ×ķ×ijר +פ ×ķ×¢×ľ +d ım +éĢļ ãĤĬ +ĠбÑĥд ÑĥÑĤ +à¹Ģวà¹ĩà¸ļ à¹Ħà¸ĭ +à¹Ģวà¹ĩà¸ļà¹Ħà¸ĭ à¸ķà¹Į +ا خر +×Ĺ ×Ļ׾ +Ġ×Ļ ׾ +Ġ×Ļ׾ ×ĵ×Ļ×Ŀ +×Ĺ ×Ļפ +×Ĺ×Ļפ ×ķש +Ġd òng +Ġש ×ĸ×Ķ +ÑĮ е +ãģĤ ãģ¨ +ìŀIJ ê°Ģ +×IJ ×ĵ +Ġü z +Ġüz ere +ظ ÙĦ +Ġ×IJ ×ķ׾×Ļ +Ġ×ij ×Ļ×ķ×Ŀ +ÙĦ ات +Ġm ê +ì¹ ¨ +تØŃ Ø¯ +تØŃد Ø« +ĠØ® اصة +Ġب رÙĨ +ĠبرÙĨ اÙħج +ĠH Ãłn +×Ĺ ×¡ +ĠÙĪ ÙĦÙħ +×¢ ×Ŀ +Ġm ı +à¸Ł ัà¸ĩ +ש ×¢×Ķ +ÙĪÙģ ÙĤ +ס ×ij×Ļר +алÑĮ нÑĭй +×Ĺש ×ķ×ij +Ġn Ãłng +ë³ ¼ +ĠкоÑĤоÑĢ ÑĭÑħ +Ġ×Ĺ ×ķק +t ör +ĠлÑĥÑĩ ÑĪе +ãĥij ãĥ³ +ลà¹Īา สุà¸Ķ +Ġج دÙĬد +ÙĬد Ø© +à¸Ĺ รà¸ĩ +ãĤĪãĤĬ ãĤĤ +ÙĦ ÙĦ +ãĤĤ ãģ£ãģ¨ +ש×ĺ ×Ĺ +Ġ×ķ ×IJ×Ļ +Ġgi á»ijng +Ø¥ ضاÙģ +ק ת +ë§ Ŀ +Ġzosta ÅĤ +ÑĢ оз +×Ļפ ×Ļ×Ŀ +Ġ׼׾ ׾ +ת×ķ׼ ף +dıģ ını +ÙĤ سÙħ +ĠÑģ ÑĩиÑĤ +ĠÑģÑĩиÑĤ а +×ĺ ×ķת +Ġ Æ°u +ĠØ¢ ÙĦ +Ġм ом +Ġмом енÑĤ +ĠاÙĦتع ÙĦÙĬÙħ +×¢×ľ ×ķת +Ġch ữa +Ġy ön +Ġtr Ãł +ĠØŃ ÙĬÙĨ +à¸ĭ ั +ĠC á +×¢ ×ĸ +ĠاÙĦØ£ ÙħÙĨ +c ÃŃ +Ġv á»ijn +Ġ à¸Ļาย +об ÑĢа +ק ×IJ +Ġthi ếu +ãĥŀ ãĥ¼ +ส วà¸Ļ +Ġg á»Ń +Ġgá»Ń i +Ġê ¹ +Ġê¹ Ģ +Ġthi á»ĩn +ÙĤ ع +w ÄĻ +Ġн ам +ÑĤ ол +Ġs ân +ס ×ķ×Ĵ +Ġgeç ir +ÑĤ он +ев а +ĠÙĪ ضع +Ġع شر +Ñģ ло +à¸Ī ัà¸ļ +ãĤ· ãĥ¼ +ãĤĤ ãģĤãĤĬãģ¾ãģĻ +Ġv ẻ +ĠÄIJ á»ĥ +ر Ùģع +ĠاÙĦØ£ÙĪÙĦ Ùī +ÑĤ аÑĢ +ãģªãģı ãģ¦ +Ùħ Ùİ +qu ÃŃ +×¢×ł×Ļ ×Ļ׳ +г ен +Ġh ôm +à¸Ī า +Ġnh Ỽ +ĠاÙĦع ربÙĬ +×IJ ף +Ġl á»Ļ +Ġje ÅĽli +à¹Ģà¸Ĺà¹Īา à¸Ļัà¹īà¸Ļ +ĠØ£ÙĨ Ùĩا +Ġt uy +Ġtuy á»ĩt +Ġت ص +Ġتص ÙĨÙĬ +ĠتصÙĨÙĬ Ùģ +Ġê·¸ëŁ¬ ëĤĺ +о ÑĨен +à¸ģิà¸Ī à¸ģรรม +ãĤĦ ãģ£ãģ¦ +Ġkh á»ıi +Ġl á»ĩ +ĠاÙĦÙħج تÙħع +à¸Ńาà¸Ī à¸Īะ +à¸Īะ à¹Ģà¸Ľà¹ĩà¸Ļ +ов Ñĭй +ר ×Ŀ +ร à¹īà¸Ńà¸Ļ +ש ×ŀש +人 ãģ« +Ġüzer ine +פר ×Ļ +du ÄŁu +Ñĩ ик +Ġmù a +Ġ×ŀת ×ķ×ļ +Ġc áºŃp +Ġت ارÙĬØ® +×ij׾ ת×Ļ +Ġì¢ Ģ +ÙĦ ع +ب اÙĨ +Ġch út +Ġ×Ķ×ĸ ×ŀף +n ée +ĠLi ên +ĠÙĦÙĦ Ø£ +ØŃد ÙĪد +Ġ×¢ ׼ש×Ļ×ķ +в оз +Ġyapt ı +Ġоб о +à¹ĥหà¹ī à¸ģัà¸ļ +Ġ×ij×Ķ ×Ŀ +ãģı ãģ¦ +ر أس +ĠÑģÑĢед ÑģÑĤв +ĠB Ãłi +ãģĵãģ¨ ãģ« +ĠìĤ¬ íļĮ +Ġ모 ëijIJ +×ij ×IJ +Ġtr ắng +ĠاÙĦبÙĦ د +ĠHo Ãłng +ли бо +ĠдÑĢÑĥг иÑħ +Ä° R +Ñĥм а +ĠJe ÅĽli +ãĤĤ ãģĹ +Ġv òng +Ġ×IJתר ×Ļ×Ŀ +ĠÄij á»įc +Ġв оÑĤ +ãģł ãģĮ +ë° ° +à¸Ķู à¹ģล +Ġ×ŀ ׼׾ +ìĹIJ ëıĦ +г аз +Ġ׳×ķס פ×Ļ×Ŀ +ãģĵãģ¨ ãģ§ +Ġت ÙĪ +ãģ§ ãģĤãĤĬ +à¸Ļั à¹Īà¸ĩ +ĠможеÑĤ е +sz ÄĻ +ãģ® ãģł +ĠÙħÙĨ Ùĩ +Ġb á»ķ +Ġb üt +Ġbüt ün +ë³´ ê³ł +Ġch á»ĵng +à¹ģà¸Ī à¹īà¸ĩ +ĠV ì +ĠØŃ Ø± +Ġgi ản +ĠÙħ دÙĬÙĨØ© +تط بÙĬÙĤ +à¸Ī ิ +æĹ¥ ãģ® +б ил +à¸ģ à¸Ńà¸ĩ +ê³ ³ +ĠØ£ Ùħا +ìĨ IJ +Ġtr ái +ĠвÑģ ем +Ġس ÙĨØ© +ĠÑģай ÑĤ +Ġг оÑĤов +п Ñĭ +ĠëIJ ł +ĠاÙĦØ® Ø· +ĠاÙĦرئÙĬس ÙĬØ© +Ġíķ ©ëĭĪëĭ¤ +ĠìķĦëĭĪ ëĿ¼ +ĠìĿ´ ëłĩ +ĠìĿ´ëłĩ ê²Į +) ØĮ +h ält +ĠØ£ Ùħر +Ġع Ùħر +à¸ģà¹ĩ à¸Īะ +Ġ à¸Ĺำà¹ĥหà¹ī +Ġc ân +Ġ×ij ׾ +Ġ×ij׾ ×ij×ĵ +פ סק +ĠÙĬ ÙĤÙĪÙĦ +н ÑĥÑĤÑĮ +à¹ģ à¸Ħ +Ġק צת +Ġn ằm +Ġh òa +bilit Ãł +ĠìĹĨ ëĭ¤ +Ġ׼ פ×Ļ +ÑĢ ож +лаг а +Ġ×Ķש ×Ļ +ĠNgo Ãłi +ĠÙĪ ج +ĠÙĪج ÙĪد +ĠìľĦ íķľ +Ġus ÅĤug +Ġtu ần +d ź +×ŀ ×ķף +ĠاÙĦع دÙĬد +Ġch ẳng +สุà¸Ĥ à¸łà¸²à¸ŀ +Ġ×ij ×ĵר×ļ +ĠÑģеб е +ĠìŀĪ ìĿĦ +ĠاÙĦØŃ Ø§ÙĦ +Ġd á +Ġc Æ°á»Ŀi +Ġnghi ên +ie ÅĦ +ĠD Æ°Æ¡ng +ï¼ ħ +Ø´ د +ãģĦãģ¤ ãĤĤ +ĠвÑĭб оÑĢ +Ġc á»Ļng +ש ×Ļ׳×ķ×Ļ +Ġch ạy +Ġ×ij×¢ ׾×Ļ +اخ بار +íķĺ ë©° +ż Äħ +ج از +Ġ׳ ר×IJ×Ķ +ศ ู +ศู à¸Ļ +ศูà¸Ļ ยà¹Į +×Ĵ ×¢ +Ġ×¢ ×ĵ×Ļ +Ġ×¢×ĵ×Ļ ×Ļף +بر ا +ÑĨи й +ĠÄIJ á»ĵng +ÙĤ اÙĨÙĪÙĨ +ĠÄij ứng +ãģĹãģŁ ãĤĬ +Ġ×Ĺ×Ļ ×Ļ +Ġë IJľ +ĠëIJľ ëĭ¤ +Ġм еждÑĥ +à¸ŀวà¸ģ à¹Ģà¸Ĥา +ĠB ắc +ล ำ +ë° ± +ĠíĻ ķ +มาà¸ģ ม +มาà¸ģม าย +бан к +à¸Ńา à¸ģาร +Ġh Ãł +Ġ׾ ׳ +à¸Ń à¸Ń +Ġë°Ķ ë¡ľ +л ом +m ática +ĠØŃ Ø¯ +اب ت +à¸Ĺีà¹Ī à¸Ļีà¹Ī +Ġco ÅĽ +ÙģÙĬ دÙĬ +ÙģÙĬدÙĬ ÙĪ +ĠмеÑģÑĤ о +Ġph út +มาà¸ģ à¸ģวà¹Īา +×IJ פ +ب ÙIJ +ĠPh ú +ì± Ħ +ĠÙĪ سÙĦÙħ +à¸Īี à¸Ļ +поÑĤ ÑĢеб +Ġ×Ĺ×ĵ ש×ķת +Ø´ ÙĪ +Ġעצ ×ŀ×ķ +ĠعÙħÙĦ ÙĬØ© +à¸Ħุà¸ĵ à¸łà¸²à¸ŀ +ãģ¾ãģĻ ãģĮ +دع ÙĪ +طر ÙĤ +à¹Ħมà¹Ī à¸ķà¹īà¸Ńà¸ĩ +ë² Ķ +ìĬ ¹ +Ġk ÃŃch +ĠìĹĨ ëĬĶ +ĠÑĤ ам +ĠÙĨ ØŃÙĪ +ĠاÙĦÙĤ اÙĨÙĪÙĨ +×Ĺ ×ķ×Ŀ +Ġk ız +Ġ×ĵ ×Ļף +ĠвÑĢем ени +ãģ£ãģŁ ãĤĬ +ĠØ´ Ùĩر +ĠìĦľ ë¹ĦìĬ¤ +×¢ ש×Ķ +Ġgi ác +ĠاÙĦسÙĦ اÙħ +Ġ×IJ ש +ĠполÑĥÑĩ а +à¸Īัà¸Ķ à¸ģาร +к оÑĢ +Ġ×Ķ×ĺ ×ķ×ij +ราย à¸ģาร +주 ìĿĺ +à¹ģà¸ķà¹Ī ละ +Ġê·¸ëŁ° ëį° +à¸Ĺีà¹Ī à¹Ģà¸Ľà¹ĩà¸Ļ +Ġת ×ķ×ļ +بÙĬ اÙĨ +Ð Ļ +oÅĽci Äħ +ÑĤ ок +Ġà Ķ +ĠÃĶ ng +à¹Ħมà¹Ī à¹ĥà¸Ĭà¹Ī +ãģ¿ ãģ¦ +ÐŁ о +ĠЧ ÑĤо +íĻ © +×ĺ ×ij×¢ +меÑĤ ÑĢ +Ġ×ij ×ŀ×Ķ +Ġ×ij×ŀ×Ķ ׾ +Ġ×ij×ŀ×Ķ׾ ×ļ +Ñĩ ÑĮ +ק ש×Ķ +з нак +знак ом +uj ÄĻ +×Ļצ ר +ĠاÙĦÙħ ÙĦÙĥ +ı yla +×IJ×ŀ ת +à¸Ľ ิà¸Ķ +×IJ ×Ĺ×ĵ +ر اد +Ġm áºŃt +ëĭ¤ ëĬĶ +Ġl ạnh +ש׾ ×ķש +ØŃ Ø¯ÙĬØ« +ت ز +å¹´ ãģ® +Ġк ваÑĢ +ĠкваÑĢ ÑĤиÑĢ +ä½ľ ãĤĬ +رÙĪ ب +ов ан +ĠТ е +à¸Īำ à¸ģ +à¸Īำà¸ģ ัà¸Ķ +ب اط +×Ĵ ת +Ġм аÑĪ +ĠмаÑĪ ин +×Ļצ ×Ķ +ãģ» ãģ¨ +ãģ»ãģ¨ ãĤĵãģ© +ÃŃ do +ĠÑı зÑĭк +à¸ļ ิà¸Ļ +สà¸ĸาà¸Ļ à¸Ĺีà¹Ī +ĠìĹ ´ +ãĤ¦ ãĤ§ +Ġc Ãł +п ан +åı£ ãĤ³ãĥŁ +Ġر د +اÙĤ ت +ĠÙĥ ب +ĠÙĥب ÙĬرة +ÑģÑĤ ал +ש×ŀ ×Ĺ +pos ición +ĠÙħÙĦÙĬ ÙĪÙĨ +ĠìĿ´ ìķ¼ +ĠìĿ´ìķ¼ 기 +Ġh út +ĠÅĽw iat +Ġë°© ë²ķ +ĠÑģв еÑĤ +Ġвиде о +ĠاÙĦÙĨ ظاÙħ +Ġtr á»Ŀi +ĠëĮĢ íķ´ìĦľ +ר ×ŀת +ت داÙĪÙĦ +×ķר ×ĵ +ת ×ŀ +ת×ŀ ×ķ׳×ķת +Ġ×ŀ ף +Ġдв а +Ġ×Ķק ×ķ +æĹ¥ ãģ« +Ġ×Ķ×Ĵ ×Ļ×¢ +à¹Ģà¸ŀิà¹Īม à¹Ģà¸ķิม +Ùħار س +Ġê²ĥ ìŀħëĭĪëĭ¤ +ãģªãģĦ ãģ¨ +Ġnhi á»ĩt +ëIJ ©ëĭĪëĭ¤ +Ġ×ij׳ ×ķש×IJ +Ġê°Ģ ìŀ¥ +Ġv ợ +ĠÄij óng +צ×Ļ׾ ×ķ×Ŀ +ê´Ģ ê³Ħ +в аÑı +×IJ ×Ļ×ĸ +×IJ×Ļ×ĸ ×Ķ +ĠÙĨ ظاÙħ +ÙħØŃ Ø§Ùģظ +Ġt ải +기 ëıĦ +à¸Ľà¸±à¸Ī à¸Īุ +à¸Ľà¸±à¸Īà¸Īุ à¸ļัà¸Ļ +׼ ×ĵ×ķר +ĠìķĦ ìĿ´ +׼׳ ×Ļס +à¹Ģ à¸ķร +à¹Ģà¸ķร ียม +Ġngo ại +ĠدÙĪÙĦ ار +Ġr ẻ +Ġkh Äĥn +عد د +Ø´ عب +czy Äĩ +ĠاÙĦ Ùĥر +ĠÑĩеловек а +ĠÙĪ Ø¥ÙĨ +×IJ ×ĺ +Ġth Æ¡ +ĠاÙĦ رÙĬاض +оп ÑĢедел +опÑĢедел ен +×Ķ ×ŀש×ļ +ĠÐĿ ово +з Ñĭва +ĠاÙĦدÙĪÙĦ ÙĬ +ĠÄij áp +Ġк ÑĢед +ĠкÑĢед иÑĤ +ов ого +Ġm ôn +à¸Ľà¸£à¸° à¹Ĥย +à¸Ľà¸£à¸°à¹Ĥย à¸Ĭà¸Ļ +à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļ à¹Į +ÑģÑĤ е +ĠTh á»ĭ +د ÙĬØ© +×ŀצ ×ķ +Ùģ ات +ק ×ĵ×Ŀ +ìĿ´ëĿ¼ ê³ł +ÙĪ Ø® +Ġ×Ĺ ×ĸ +ĠÑĦоÑĤ о +׾ ×Ļת +ت Ùİ +ÙĪ بر +й ÑĤи +ĠÃ¶ÄŁ ren +Ġ×Ķ×ĸ ×ķ +Ġv á»įng +ÙĤÙĪ Ø© +ĠT ây +ĠÐĿ и +Ġש ×ķ×ij +ãģ¨è¨Ģ ãĤıãĤĮ +ãģ© ãĤĵãģª +׊צ×Ļ +ï½ ľ +Ġ×ķ×Ķ ×ķ×IJ +ä¸Ģ ãģ¤ +ĠÑģÑĤо иÑĤ +ni Äħ +×ĺר ×Ļ +ĠдеÑĤ ей +нÑı ÑĤÑĮ +ĠÑģдел аÑĤÑĮ +Ġë§İ ìĿ´ +ä½ķ ãģĭ +ãģĽ ãĤĭ +à¹Ħ หม +à¸ķิà¸Ķ à¸ķà¹Īà¸Ń +Ġ×ij ת×Ĺ +Ġ×ijת×Ĺ ×ķ×Ŀ +ìĻ Ħ +ì§Ģ ëĬĶ +ÑģÑĤ аÑĤ +ÑıÑģ н +ü b +Ġth ả +Ġ×ij×IJ×ŀ ת +Ġt uyến +×ĵ ×Ļר×Ķ +Ġ×IJ ×Ļש×Ļ +×ĸ׼ ר +ãģ° ãģĭãĤĬ +Ġx ét +׼ ×Ļ×ķ +׼×Ļ×ķ ×ķף +diÄŁ ini +ĠاÙĦÙħ ÙĪضÙĪع +Ġh áºŃu +à¸Īาà¸ģ à¸ģาร +×ijס ×Ļס +Ġ×ŀ×Ĵ ×Ļ×¢ +×ij ×Ļ×¢ +ĠÙĪ جÙĩ +à¹ģà¸Ķ à¸ĩ +à¸Ļ าà¸ĩ +ĠÅŀ a +ì ¡´ +ë¡ Ģ +à¸ķ ะ +Ġ×Ķ×Ĺ×Ļ ×Ļ×Ŀ +Ùģ ÙĬد +ãģ§ãģĻ ãģĭãĤī +ê· ľ +ź ni +ĠлÑİ Ð´ÐµÐ¹ +Ġyüz de +ıy orum +ĠاÙĦ بØŃر +e ño +п аÑĢ +ÙĬ ÙĤØ© +об ÑĢ +ר ×ķ×ļ +ت ÙĪÙĤع +ĠاÙĦØ´ ÙĬØ® +åĪĿ ãĤģãģ¦ +ĠÑĤ елеÑĦ +ĠÑĤелеÑĦ он +Ġth ôi +Ġ×Ļ׼×ķ׾ ×Ļ×Ŀ +ĠÅŁ irk +ĠÅŁirk et +Ġìļ°ë¦¬ ê°Ģ +ĠÄij ông +Ġת ×ķ×ĵ×Ķ +ÑģмоÑĤÑĢ еÑĤÑĮ +ĠÙĦ ÙĩÙħ +Ġ׾ ׼ +ĠN ó +ĠØŃ Ø§ÙĦØ© +ãģĦ ãģij +קר ×ķ +az ı +ãĤ³ ãĥ¼ +ĠÙĦÙĦ ت +s ınız +ĠH ải +기 ìĪł +ยัà¸ĩ à¹Ħมà¹Ī +ëĭ¤ ê³ł +פ ×Ĺ +Ġ׾×Ĵ ×ij×Ļ +Ġع ÙĨÙĩ +Ġк аз +Ġказ ино +ب ÙĪر +ÑĦ еÑĢ +Ġê°Ļ ìĿ´ +تس جÙĬÙĦ +ĠاÙĦÙħ رÙĥز +ĠTh ái +д аÑĤÑĮ +×ŀ×Ļ ×Ļ׾ +Ġpay laÅŁ +ãģ¤ ãģ® +à¹Ģร ืà¸Ń +n ça +׳ ×ķ×Ĺ +Ġ×IJ פ×Ļ׾×ķ +ãģ¨ èĢĥãģĪ +ãģ¨ãģĹãģ¦ ãģ¯ +à¹Ģà¸Ī à¸Ń +×ŀ פ +Ġg iriÅŁ +л иÑĤ +ÑĤ елÑı +Ñij н +æ°Ĺ ãģ« +Ġg ó +Ġgó p +åĪĩ ãĤĬ +Ġ×Ķ ×Ĺ×ĵש +ж ал +Ġ×ĵ עת +éģķ ãģĨ +à¹Ģà¸Ĥà¹īา à¹Ħà¸Ľ +Ġס ר×ĺ +e ña +æĸ° ãģĹãģĦ +ر Ùİ +ĠÐIJ ÑĢ +Ġph ản +à¸Īะ à¹Ħà¸Ķà¹ī +Ġ×ijצ ×ķר×Ķ +Ø´ اÙĩ +شاÙĩ د +ÙĪر د +à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ à¸Īาà¸ģ +или ÑģÑĮ +à¹ģละ à¸ģาร +Ġ×Ķ ×ĸ׼ +Ġ×Ķ×ĸ׼ ×ķ×Ļ×ķת +ei ÃŁ +ãĥ ¨ +ìĥ Ī +ĠÃĩ a +Æ ¯ +ש ×Ĵ +ÙĬÙĨ Ø© +ร à¹īà¸Ńà¸ĩ +ãĤµ ãĥ³ +ÑĢоÑģÑģ ий +ÑĢоÑģÑģий Ñģк +a ÄŁa +ĠнаÑĩ ина +Ġص ÙĦÙī +à¸Ĺุà¸ģ à¸Ħà¸Ļ +íļĮ ìĤ¬ +Ġли ÑĨ +Ø´ ÙĬر +ĠØ´ÙĬ Ø¡ +ÙĬÙĨ ا +Ġפ ×Ĺ×ķת +Ġiçer is +Ġiçeris inde +ĠØ£ ØŃÙħد +Ġże by +ì´ Ŀ +Ġп оказ +Ġи менно +หà¸Ļัà¸ĩ ส +หà¸Ļัà¸ĩส ืà¸Ń +ĠÑĤÑĢ е +สัà¸ĩ à¸Ħม +Ø¥ ÙIJ +ãģĮ å¿ħè¦ģ +ÙĬÙij Ø© +פ צ +íĭ ° +ĠÙħ جاÙĦ +׳ פש +к ан +×Ĺ ×ķפ +×Ĺ×ķפ ש +ì²ĺ ëŁ¼ +ов аÑı +з ов +Ġh ạ +Ġdzi ÄĻki +×Ļר ×ķ +Ġ׾ ×ŀצ +Ġ׾×ŀצ ×ķ×IJ +×Ļ×ĵ ×ķ +Ġs ợ +Ġ׾×Ķ ×Ĵ×Ļ×¢ +ק ×ij×¢ +Ġchi á»ģu +ãĥŀ ãĤ¤ +Ġd Ãłng +à¹ģà¸Ł à¸Ļ +Ġü ye +×Ļ׳ ×Ĵ +à¹Ģรีย à¸ģ +ç§ģ ãģĮ +th é +ĠÑĦ илÑĮ +ĠÑĦилÑĮ м +ĠNg Ãły +Ġж ен +Ġжен Ñīин +ج ÙĬد +n ç +à¸Ľ รา +×Ļ×ŀ ×ķ +Ġn á»ģn +×IJ ×ķ׾×Ŀ +Ġвозмож ноÑģÑĤÑĮ +Ġëĭ¤ ìĭľ +è¦ĭ ãģŁ +à¸ĸ à¸Ļ +à¸ĸà¸Ļ à¸Ļ +mız ı +ĠÙħ جÙħÙĪعة +c jÄħ +ĠÐł Ф +à¸ģำ หà¸Ļ +à¸ģำหà¸Ļ à¸Ķ +ĠìŬ 기 +land ı +ни ÑĨ +ÑģÑĤв е +Ġ×ĵ ×ijר×Ļ×Ŀ +Ġsk ÅĤad +ãĤĬ ãģ¾ãģĹãģŁ +ĠоÑĤ кÑĢÑĭÑĤ +нÑı ÑĤ +ĠÑģво ей +à¸Ī ิà¸ķ +ĠкаÑĩеÑģÑĤв е +Ġet tiÄŁi +ìĤ¬ íķŃ +ĠاÙĦÙĬ ÙħÙĨ +иÑĩеÑģки й +ë¸ Į +Ġ×ij×IJר ×¥ +Ġا سÙħ +Ġиз веÑģÑĤ +r ão +Ġatt ivitÃł +à¹Ģà¸Ľà¹ĩà¸Ļ à¸ģาร +ĠاÙĦد Ùĥت +ĠاÙĦدÙĥت ÙĪر +ĠÙĪاØŃد Ø© +ĠÑģ ÑĩеÑĤ +ĠпÑĢ иÑĩ +ĠпÑĢиÑĩ ин +ĠÙĪز ارة +Ġh uyá»ĩn +ĠÙĥ تاب +à¹ģà¸Ļ à¹Īà¸Ļ +à¹ģà¸Ļà¹Īà¸Ļ à¸Ńà¸Ļ +Ġgün ü +г ÑĢÑĥз +ĠاÙĦØ® اص +Ġgör ül +׾ ×ŀ×ĵ +Ġìłķ ëıĦ +×ķ×ij ×Ļ׾ +Ġ×ŀק צ×ķ×¢×Ļ +ĠоÑģоб енно +à¸Ľà¸£à¸° à¸ģา +à¸Ľà¸£à¸°à¸ģา ศ +aca ģını +ë¶ ģ +à¸łà¸¹ มิ +ĠÑį лекÑĤ +ĠÑįлекÑĤ ÑĢо +Ġק ש×Ķ +سÙĦ Ø· +à¸Ĭà¸Ļ ะ +×¢ ×Ļ׾ +ĠЧ е +à¹ģà¸Ļ à¹Ī +lı ÄŁ +lıģ ın +Ġ×ŀ×¢ ×¨×Ľ×ª +好ãģį ãģª +มาà¸ģ à¸Ĥึà¹īà¸Ļ +×ŀ×¢ ×ijר +ĠاÙĦÙħ غرب +ĠпеÑĢ и +ĠпеÑĢи од +Ġnh ạc +ا ÙĪÙĬ +ĠÙĪ عÙĦÙī +أخ Ø° +ĠC ô +תר ×ij×ķת +×Ĵ ×Ķ +Ġktóre j +×IJ ×Ļת +×ij ×ķ×IJ +д елÑĮ +รี วิ +รีวิ ว +ж Ñĥ +Ġ×ij×Ĺ ×ķ +еÑĪ ÑĮ +ĠØ£ ÙĦÙģ +ĠاÙĦÙĪ Ø·ÙĨÙĬ +ĠاÙĦÙħÙĨ Ø·ÙĤØ© +nÄħ Äĩ +Ġthi ên +иÑĩеÑģк ой +ĠاÙĦÙħ ÙĦ +Ġع Ùħ +ס פר +Ġnh óm +ÙĪص Ùģ +ĠCh úng +Ġر ÙĤÙħ +ãģ¾ãģĹãģŁ ãģĮ +al ité +ล ม +ĠëĤ´ ê°Ģ +׾ק ×ķ×Ĺ +ĠS Æ¡n +pos ição +mi ÄĻ +Ġtr ánh +ĠÄIJ á»Ļ +׼ ×Ĺ +ãģĤ ãģ£ãģ¦ +à¸Ńย à¹Īา +Ġ×ŀ×Ĺ ×Ļר +Ġ×Ķ ×Ļת×Ķ +à¸Ľ à¹Īา +à¸Ńืà¹Īà¸Ļ à¹Ĩ +Ø´ ÙĤ +×ł×¡ ×Ļ +ë¦ ¼ +ãģ¦ãģĹãģ¾ ãģĨ +Ġ×ŀ צ×ij +ãģ« åĩº +ÙħÙĪا Ø·ÙĨ +ยัà¸ĩ มี +алÑĮ нÑĭе +san ız +Ø¥ سرائÙĬÙĦ +ĠvÃł i +ì¤ Ħ +ãģ¨æĢĿ ãģ£ãģ¦ +×Ļ ×ķ׳×Ļ +çĶŁ ãģį +Ġs âu +Ñĩ иÑģÑĤ +Ġl á»ħ +ĠGi á +à¸Ńุ à¸Ľ +à¸Ńà¸¸à¸Ľ à¸ģร +à¸Ńà¸¸à¸Ľà¸ģร à¸ĵà¹Į +Ġnh ẹ +r ö +ס ×ĺ×Ļ +ãģķãĤĵ ãģĮ +Ġd ầu +ع Ùİ +ت را +×Ĵ×ĵ ׾ +Ġtécn ica +׼ ׳×Ļ×Ŀ +תק ש +תקש ×ķרת +Ġн его +ét ait +Ġm á»ģm +Ñģ еÑĤ +Ġnh áºŃt +Ġ×ŀ ×¢×ľ +Ġ×Ķ×¢ ×ij×ķ×ĵ +Ġ×Ķ×¢×ij×ķ×ĵ ×Ķ +Ġ×Ĵ ×Ļ׾ +ãģ¯ ãģªãģĦ +ائ ØŃ +Ġз деÑģÑĮ +×IJ ×Ļ׳×ĺר +Ùħ ÙIJ +Ġ×Ļ ×Ĺ×ĵ +ر اÙģ +ì²ĺ 리 +×ĵ ×¢×ķת +ì¹ ľ +ĠТ о +ĠTh ế +ì¶ © +Ġ׳׼ ×ķף +عÙĬ Ø´ +ни з +Ġج اÙĨب +×ŀק צ×ķ×¢ +à¹Ĥ à¸ĭ +Ñģ ÑĥÑĤ +ìĸ´ ìļĶ +ãĤĴè¦ĭ ãģ¦ +ار د +Ġaç ıl +ĠاÙĦØŃ ÙĬاة +à¸ģà¹ĩ à¹Ħà¸Ķà¹ī +ãģĿãĤĮ ãĤĴ +عض ÙĪ +Ġг ÑĢаж +ĠгÑĢаж дан +à¸Īะ à¸ķà¹īà¸Ńà¸ĩ +ĠìĿ´ 룬 +ĠìĿ´ëŁ¬ íķľ +Ġtr ách +ÙĨ Ùİ +Ġkı sa +à Ķ +ÑĪ ка +ãģ® 人 +ĠÐŁ оÑģ +ĠÐŁÐ¾Ñģ ле +Ñĥ лÑĮ +ÙĪا جÙĩ +ÙĤ رب +à¸Ľà¸ıิ à¸ļัà¸ķิ +ê° Ļ +Ġ×ŀ ׳ +ĠÑģво и +بر اÙħج +Ġر ÙĪ +пÑĢ од +пÑĢод аж +Ġby ÅĤy +วั ย +Ġgör ün +Ġà Ī +ÑİÑī им +ĠÑĤак ой +Ùģ ÙĪر +ĠÙģ عÙĦ +Ġб ел +ëIJ ł +er ÃŃa +ĠÑģво Ñİ +Ġl ã +Ġlã nh +à¹Ģà¸ŀืà¹Īà¸Ń à¹ĥหà¹ī +ÙĤ ÙĨ +تط ÙĪÙĬر +Ġsay ı +ĠÑģ ейÑĩаÑģ +Ġ×IJ×Ĺר ת +ק ×ķפ×Ķ +ק×ķר ס +Ġس Ùħ +Ġ×ĺ ×Ļפ×ķ׾ +ìĿ´ëĿ¼ ëĬĶ +دراس Ø© +èµ· ãģĵ +×Ĺ ×Ļ׳ +×Ĺ×Ļ׳ ×ķ×ļ +×ĵ ק +Ġë§ ŀ +Ġком анд +ĠÐij о +Ġиг ÑĢÑĭ +à¸ļ ี +ĠØ£ Ùİ +в ен +ĠاÙĦج دÙĬد +ĠÙĦ Ø¥ +Ġ×ķ×IJ ׳×Ļ +Ġ×Ķס ×Ļ +иÑĩеÑģк ого +رÙĪ ØŃ +à¸ģาร ศึà¸ģษา +ĠTr Æ°á»Ŀng +иг ÑĢа +ıl ması +Ġм аÑģÑģ +ãģ¨ãģį ãģ« +à¸Ĺีà¹Ī à¸ľà¹Īาà¸Ļ +à¸Ĺีà¹Īà¸ľà¹Īาà¸Ļ มา +ĠاÙĦساب ÙĤ +Ġ×ŀ×¢ ×ĺ +в аÑĤÑĮ +m Ã¼ÅŁ +Ġ׾ ׼×ļ +Ġt á»ĭch +Ùģ ÙĩÙħ +تد رÙĬب +Ø´ Ùĥ +Ġ×ij ×ŀ×Ļ +Ġ×ij×ŀ×Ļ ×ķ×Ĺ×ĵ +ÙĤØ· اع +ãģª ãģĹ +×ķצ ×Ļ×IJ +ĠÙĪ سÙĬ +з Ñĥ +Ġy at +Ġyat ırım +ë§ İ +Ġth ắng +ãģĬ 客 +ãģĬ客 æ§ĺ +ĠThi ên +ãģ«å¯¾ ãģĹãģ¦ +ÑĢ иÑģ +ÙĨت ائ +ÙĨتائ ج +Ġ×ŀ שר +Ġ×ŀשר ×ĵ +Ġتع اÙĦ +ĠتعاÙĦ Ùī +ש ׳×Ļ +Ùĩ اÙħ +×IJ׳ ש×Ļ×Ŀ +Ġżyc ia +ĠÑĢÑĥб лей +ÙĬ ض +Ġkat ıl +ĠÙħ ÙĪضÙĪع +Ġvard ır +ĠÙħÙĨ Ø·ÙĤØ© +ĠTr ần +Ġв еÑģ +ü p +Ùħ ÙĪÙĨ +ÑĪ ли +Ġn óng +Ø® ÙĦÙģ +ĠС ÑĤа +Ġд оÑĢ +ĠдоÑĢ ог +ĠwÅĤa ÅĽnie +eÄŁ in +Ġhi á»ĥm +ĠС ам +ê»ĺ ìĦľ +ĠÑĦ а +ãģ» ãģĨ +ãģ»ãģĨ ãģĮ +×ķפ ×Ļ×¢ +ê° Ī +د ÙĪÙĦ +Ġthu ê +Ġch á»Ĺ +Ġëĭ¹ ìĭł +ãģij ãĤĮ +ãģijãĤĮ ãģ© +ë³´ íĺ¸ +ãģķãĤĮ ãģ¦ãģĦãģ¾ãģĻ +Ġнад о +ĠìĤ¬ëŀĮ ëĵ¤ +à¹Ģà¸Ĥ à¸ķ +สม ัย +z ÅĤ +ت ÙĪر +Ġש ת×Ļ +v ê +Ġ×ijת ×ķ×ļ +à¸Ĭ ัย +ãģĦ ãģ£ãģŁ +ìĿ ij +Ġt ầ +Ġtầ ng +ש ׼ר +Ġê¸ Ģ +Ġ×Ķש ׳×Ķ +Ġا ÙĨÙĩ +ç«ĭ ãģ¡ +r és +füh ren +ر ØŃÙħ +ê· ¹ +ĠâĢ « +Ġsu ất +à¸Ł ิ +ÙĬ Ùĩا +ĠاÙĦ اتØŃاد +Ġt uyá»ĥn +ãģ¾ ãĤĭ +Ġm ại +Ġng ân +ãĤ° ãĥ© +欲 ãģĹãģĦ +س ار +ãĤĤãģ® ãģ§ãģĻ +ки е +Ġseç im +åħ¥ ãĤĬ +ãģªãģ© ãĤĴ +ÑĤ ÑĢи +ĠÑģп еÑĨ +ĠØ£ د +Ġод но +ÑĪ ел +ãĥĩ ãĥ¼ãĤ¿ +ãĤ· ãĤ¹ãĥĨ +ãĤ·ãĤ¹ãĥĨ ãĥł +è¡Į ãģį +ãģ¨æĢĿ ãģ£ãģŁ +à¹Ģà¸ģิà¸Ķ à¸Ĥึà¹īà¸Ļ +ĠÑĤ ож +ĠÑĤож е +Ġs ạch +ĠÑģ ÑĢок +Ġкли енÑĤ +ĠÙħØ´ رÙĪع +Ġalt ında +Ġì ·¨ +ä¸Ń ãģ® +ãģķãģĽ ãĤĭ +ãģĻ ãģ¹ +ãģĻãģ¹ ãģ¦ +ê°ľ ë°ľ +ĠÄij êm +ãģªãģĦ ãģ®ãģ§ +ì² ł +×¢ ×ij×ĵ +Ġd ấu +à¸Ħà¸Ļ à¸Ĺีà¹Ī +ĠC ách +تع ÙĦÙĬÙħ +Ġh ại +ãĤ» ãĥķãĥ¬ +ĠÙĨÙģس Ùĩ +ĠíĨµ íķ´ +ÑĪ ло +Ġнап ÑĢав +ĠнапÑĢав лен +ÑĢÑĥ Ñĩ +íĶ Į +Ġ×ijר ×Ļ×IJ +ãģ® ãģ¿ +ãģ«ãģĬ ãģĦãģ¦ +×ij ׳ק +ãĤ¨ ãĥ³ +Ø«ÙĦ اث +Ġm ỹ +ĠÑģай ÑĤе +Ġе мÑĥ +ت غÙĬ +تغÙĬ ÙĬر +خص ÙĪص +ÑĤе ли +Ġ×ķ׾ ׼ף +פע ×Ŀ +Ġпо ÑįÑĤомÑĥ +ر اÙĨ +иÑĤел ей +пиÑģ ан +×¢ ×¥ +ĠìĤ¬ ìĹħ +Ùħ ز +جÙħ ÙĬع +ë©´ ìĦľ +à¸ľà¸¥à¸´à¸ķ à¸łà¸± +à¸ľà¸¥à¸´à¸ķà¸łà¸± à¸ĵ +à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵ à¸ij +à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵà¸ij à¹Į +ĠпÑĢ имеÑĢ +ãĤŃ ãĥ¼ +l â +Ġch Äĥm +缮 ãģ® +ãģĦ ãģĭ +ãģ¨è¨Ģ ãģĨ +×ĸ ×ķ×Ĵ +Ġ×ij ×ĵ×Ļ +Ġ×ij×ĵ×Ļ ×ķק +ãģĬ åºĹ +à¸ķà¸Ńà¸Ļ à¸Ļีà¹ī +Ġph á»iji +п ÑĤ +สà¸Ļ าม +Ø· ÙĪ +ص اØŃ +صاØŃ Ø¨ +ĠD ü +ĠDü nya +Ġп ока +п ал +ĠÄij ảo +ĠاÙĦÙģ ÙĪر +ĠاÙĦÙģÙĪر Ùĥس +Ġmá u +кÑĢ еп +ĠاÙĦس اعة +ĠгоÑĢ ода +Ùģ صÙĦ +ай ÑĤе +Ġд ог +Ġдог овоÑĢ +ĠØ¥ Ø° +Ġ×ij׼׾ ׾ +ÙĬ تÙĩ +×Ĵ ×ijר +Ġbir ç +Ġbirç ok +문 íĻĶ +ãģĿãģĨ ãģª +را ØŃ +ĠÙħ رة +ĠденÑĮ ги +f ä +à¸Ĥà¹īา ว +ĠÑģов ÑĢем +ĠÑģовÑĢем енн +׾×Ĺ ×¥ +èī¯ ãģı +ĠÙģ Ø£ +Ġ×ķ ×ĸ×Ķ +Ġз ани +Ġзани ма +Ġê°Ģì§Ģ ê³ł +Ġh Æ¡i +ãģªãģ® ãģĭ +ãĥĨ ãĥ¬ãĥĵ +Ġר ×ij×ķת +à¸ķ ี +Ġ×ijש ×ł×ª +ĠT ại +Ġthu áºŃn +Ñģ ел +Ñij м +dzi Äĩ +ĠÑģ ка +ĠÑģка Ñĩ +ĠÑģкаÑĩ аÑĤÑĮ +×ķ×ŀ ×ķ +г ла +Ġмин ÑĥÑĤ +åĩº ãģĻ +Ġ×Ĺ×Ļ ×Ļ×ij +Ġת ×Ĵ×ķ×ij×Ķ +à¸£à¸¹à¸Ľ à¹ģà¸ļà¸ļ +ни ÑĨа +ĠÄ° n +ĠØ£ ع +Ġض ÙħÙĨ +Ùħ ثاÙĦ +ĠyaÅŁ an +ĠìĹ° 구 +ĠL ê +ש׾ ×Ĺ +ãģı ãģªãĤĭ +ìĹĨ ìĿ´ +ĠÑĤ ÑĢи +ĠÑĩаÑģÑĤ о +Ġоб ÑĢаÑĤ +п ло +د Ø® +دخ ÙĪÙĦ +س Ùĩ +à¸Ń าà¸ģ +à¸Ńาà¸ģ าศ +Ġ׼ ×ĸ×Ķ +Ġ×Ķ×¢ סק +ĠاÙĦØ£ ÙĨ +å¹´ ãģ« +×¢ ש×ķ +Ġש ×¢×ķת +Ġm Ãłn +×IJר ×Ļ +sı yla +Ùģر ÙĤ +ни Ñħ +Ġت ست +è¦ĭ ãģ¦ +ØŃا ÙĪÙĦ +×IJ ×Ļ׼×ķת +ĠbaÅŁ ladı +st Äħ +stÄħ pi +à¸Ĺีà¹Ī à¹Ģรา +ÙĤر ر +ج اب +Ġ×ijר ×ķר +à¹Ģà¸Ĥà¹īา à¹ĥà¸Ī +×ŀ׊קר +al ım +Ġס ×Ļפ×ķר +ãģ§ãģĤ ãĤĮãģ° +Ġש×ŀ ×ķר×ķת +Ġ×ķ ×ŀ×Ķ +ãģĵ ãģĿ +id ée +ä¸ĭ ãģķãģĦ +تÙĨا ÙĪÙĦ +Ġ ลà¹īาà¸Ļ +Ġìļ°ë¦¬ ëĬĶ +اÙĨ ا +ÑģÑĤ ой +б оÑĤ +ĠyaÅŁ am +kö y +Ø¥ ÙĦ +ÑĢ Ñĭв +기 ìĹħ +Ġ×Ķ×ŀ ×ĵ +Ġ×Ķ×ŀ×ĵ ×Ļ׳×Ķ +د ب +×¢ ×Ļ׳×Ļ +×ŀ ת×Ĺ +Ġפ ר×Ļ +ãĥĭ ãĥ¼ +اÙħ ÙĬ +Ġnh ằm +ãĤĮ ãģªãģĦ +ت عرÙģ +Ġë§Ī ìĿĮ +ìĵ ° +Ġh ấp +ר×Ĵ ×Ļ׾ +ب Ùİ +Ġr Äĥng +gl Äħd +ĠÑģиÑģÑĤем Ñĭ +Ġkh óa +ãģ§ãģĻ ãĤĪãģŃ +大ãģį ãģı +기 를 +Ġké o +ÙĪ Ø¡ +ج اÙħ +جاÙħ ع +Ġ×¢ ×Ļצ×ķ×ij +t éri +Ġת ש +Ġ×IJ ×ij×Ļ +ĠCh Æ°Æ¡ng +à¸ļริ à¹Ģว +à¸ļริà¹Ģว à¸ĵ +ãģ¤ ãģı +Ġ×Ĺ ×ķ׾ +עת ×Ļ×ĵ +ש ×Ļ×ŀ×Ķ +ëĤ ¨ +Ġש×IJ ×Ļף +ĠÙĪاÙĦ Ø¥ +ÑĦ а +Ġkh ám +Ġ×ĺ ×ķ×ij×Ķ +ĠвÑĭ Ñģ +ĠвÑĭÑģ око +ĠاÙĦØŃ Ø¯ÙĬØ« +人 ãĤĤ +d Ã¼ÄŁÃ¼ +×Ļ×Ĺ ×ķ×ĵ +تع ÙĦÙĬ +تعÙĦÙĬ ÙĤ +l ö +تØŃ Ø¯ÙĬد +н его +ĠÑĥд об +Ġ׾ ×ŀ×Ļ +Ġר ×ķצ×Ļ×Ŀ +Ġج اء +Ġ×ij ×ĸ×ŀף +à¸Ľà¸ģ à¸ķิ +é«ĺ ãģı +à¸Ľà¸¥ า +Ġart ık +Ġbug ün +ק ׳×Ļ +Ġkho á +ĠÙħ رÙĥز +ĠìŀIJ 기 +در جة +×ŀש ר×ĵ +Ġgi ấy +Ġch óng +ק פ +ÙĬب Ø© +ĠczÄĻ sto +в али +Ùĥ ب +ìŁ ģ +ส à¸ļาย +à¸Ľà¸£à¸°à¸Ĭา à¸Ĭà¸Ļ +×Ĵ ×ķ×£ +ëŁ ī +ãģ® ãģĵãģ¨ +ล à¸Ń +Ġngh á»ī +åŃIJ ãģ© +åŃIJãģ© ãĤĤ +à¹Ħà¸Ķ à¹īà¸Ńย +à¹Ħà¸Ķà¹īà¸Ńย à¹Īาà¸ĩ +×ĵ ×¢ +ĠاÙĦت Ùī +ĠÑģов еÑĤ +Ġqual itÃł +åĩº ãģĹ +ĠÑĢÑĥк ов +ĠÑĢÑĥков од +ราย ละà¹Ģà¸Ńียà¸Ķ +ãģªãģĭ ãģªãģĭ +기 ê´Ģ +Ġ×Ĺ ×ķש +Ġ×Ĺ×ķש ×ij +л оÑĤ +à¸Ļะ à¸Ħรัà¸ļ +ק×ij ×ķצ×Ķ +Ġth ái +Ġש ×ij×Ķ +ĠÑĪ кол +ĠÙĦ ÙĥÙĦ +à¹ĥà¸Ļ à¸Ĭà¹Īวà¸ĩ +ĠÙħ ÙĥاÙĨ +ë ķĮ +Ġc ải +ĠCh ÃŃ +ÑĥÑĩ а +ìĿ µ +Ġx ảy +à¸Ĭà¸Ļ ิà¸Ķ +Ġc áºŃu +к ÑĢов +ss é +ĠÙĨ ÙĪع +ĠТ а +Ø® Ùħس +פ×ķס ×ĺ +Ġm ắc +ĠÄij em +à¸ģาร à¹ĥà¸Ĭà¹ī +ר ×ķס +ĠÐĽ е +Ġth á»Ń +รà¹Īาà¸ĩ à¸ģาย +üz ü +æĹ¥æľ¬ ãģ® +ê³¼ ìłķ +ש ×Ļ×IJ +ĠìŀĪ ê³ł +×ij ×ķ׾ +ìķ ħ +ĠÙĪاÙĦ ا +ĠÐĽ и +ĠвÑģ Ñij +Ġużytk ow +×Ĺ ×ķ׾ +ر Ùģض +Ġson uç +ãģĦ ãģ¾ãģĽãĤĵ +ìĤ¬ ìĹħ +ëĪ Ħ +ÑĤ ек +Ġud ziaÅĤ +л ез +Ġ×Ķ×Ļ ×Ļת×Ļ +ãĤīãĤĮ ãģ¦ +Ùħس ؤÙĪÙĦ +ر ار +ÑĤ ан +ĠÄij Ãło +Ġר ×ķ×ij +Ġ×ijש×ij ×Ļ׾ +ä»ĬåĽŀ ãģ¯ +ãĤ¸ ãĥ¥ +Ġ×¢ ×ijר +ãģĽ ãģ¦ +п олÑĮ +ak lı +Ġk ÃŃnh +د ت +лож ение +ĠاÙĦÙħ ص +ĠاÙĦÙħص رÙĬ +à¸Īริà¸ĩ à¹Ĩ +ĠاÙĦشر ÙĥØ© +ĠÄij á»ı +ãĥĽ ãĥĨ +ãĥĽãĥĨ ãĥ« +Ñį кон +Ñįкон ом +ĠÙĪ عÙĨ +Ġת ׳ +Ġ×ª×ł ×IJ×Ļ +ĠاÙĦدÙĪÙĦ ÙĬØ© +Ġì§Ģ ìĹŃ +ãģ§ãģĻ ãģĭ +Ġв аÑĢи +ĠваÑĢи анÑĤ +ĠاÙĦع رب +ел а +Ġt Æ°á»Ľng +sk Äħ +Ġm ặc +ส ัà¸ģ +ãĥĵ ãĥ¼ +Ġ×ij ×Ĵ׾ +Ġ×ij×Ĵ׾ ׾ +ãĥķãĤ¡ ãĥ³ +×ij ×Ļצ +×ij×Ļצ ×ķ×¢ +ли ÑģÑĤ +à¸Ł ุ +à¸Łà¸¸ à¸ķ +à¸Łà¸¸à¸ķ à¸ļà¸Ńล +à¸Ŀ à¹Īาย +ìŀIJ ìĿĺ +Ġس ÙĪÙģ +Ġש ×Ķת +Ġê± ¸ +×¢ ×ij×ķ×ĵ +ãģĻãĤĭ ãģĵãģ¨ãģĮ +ĠÑĩа ÑģÑĤÑĮ +ãĤ¢ ãĥ¡ãĥª +ãĤ¢ãĥ¡ãĥª ãĤ« +Ġtak ım +Ġs Ỽ +ĠsỼ m +שר ×Ķ +è¨Ģ ãģĨ +л ан +ì» ¤ +׼ ׳×Ķ +ÙĪÙģ ÙĬ +íĹ Ī +lu ÄŁu +ĠëĮĢ íķ´ +Ġ׾×ij ×Ļת +Ġ×Ķר×IJש ×ķ׳×Ķ +ص Ùħ +Ġsö yled +Ġsöyled i +à¸Ľ าà¸ģ +Ġard ından +ãģĪ ãģŁ +à¸Ĺัà¹Īว à¹Ħà¸Ľ +Ġ׳×ķס ×£ +б олÑĮ +ãĤĵãģ§ãģĻ ãģijãģ© +ĠлиÑĪ ÑĮ +Ġ×ij ×IJ×Ļ +ĠбÑĭ ÑģÑĤÑĢо +ส ัà¸Ļ +Ġ×ij פ׳×Ļ +л еÑĩ +ĠاÙĦØ® بر +Ġsó c +Ġth ú +Ġп ÑıÑĤ +ãģĬ é¡ĺ +ãģĬé¡ĺ ãģĦ +ÑĤ ин +ãģ«ãģ¤ãģĦãģ¦ ãģ¯ +פ ף +Ġдв ÑĥÑħ +à¸į ีà¹Ī +à¸įีà¹Ī à¸Ľ +à¸įีà¹Īà¸Ľ ุ +à¸įีà¹Īà¸Ľà¸¸ à¹Īà¸Ļ +оп еÑĢ +ĠاÙĦب شر +ĠاÙĦÙħ اÙĦ +ıyor uz +تØŃ ÙħÙĬÙĦ +à¸ģ ะ +éĸĵ ãģ« +×Ĺ ×ķש +ĠNg uyên +ãģĦãģ¦ ãģĦãĤĭ +дÑĥ ÑĪ +ש פע +ÑĪ Ñĥ +å®Ł éļĽãģ« +ĠÑĢай он +ĠCh á»ī +ÙĨ صر +Ġìļ ´ +Ġìļ´ ìĺģ +Ġ×Ķ×ĵ ×Ļף +ØŃد د +ر ز +ĠاÙĦد Ùħ +ĠPh áp +ÑĤ ÑģÑı +è¦ĭ ãģĪ +Ġti á»ĥu +Ġs á»Ńa +а ÑİÑĤÑģÑı +ĠB á +Ġ×ķ ׼׾ +Ð ĸ +ÑĪ им +ìĿ´ ëĬĶ +л ев +d ık +Ġprés ente +Ġara ç +صد ÙĤ +Ġпом ог +ĠاÙĦشر ÙĤ +ĠÙĪاÙĦ Ø°ÙĬ +رÙĬ ا +×ij ׳×ķת +Ġng á»ĵi +ר ×ķפ +ר×ķפ ×IJ +Ġth ấp +ãĤĦ ãģ¯ +ãĤĦãģ¯ ãĤĬ +ĠاÙĦج دÙĬدة +éĿŀ常 ãģ« +ÙĬÙĦ ÙĬ +ìª ½ +تع اÙħÙĦ +ãģł ãģ¨æĢĿãģĦãģ¾ãģĻ +Ùħ Ùħ +иÑĤе ли +ãĤµãĤ¤ ãĤº +اد ات +ĠاÙĦÙħ اÙĦÙĬØ© +Ùĥات ب +к ли +веÑĢ Ñħ +ни Ñĩ +Ġ×ľ×¢ ×ij×ķ×ĵ +׾ ×Ļ×Ķ +ØŃ Ùİ +ãĤ¤ ãĥĻ +ãĤ¤ãĥĻ ãĥ³ãĥĪ +Ġת ×Ĵ×ķ×ij×ķת +ÑĦ он +ĠдÑĢÑĥг ие +×IJ ×ĸ×ķר +Ġper ò +ìķ ŀ +åĢŁ ãĤĬ +ר צ×Ļ +×IJ ×ĸ +алÑĮ нÑĭÑħ +Ġê²ĥ ìľ¼ë¡ľ +ĠпÑĢав о +ĠاÙĦØ£ رض +à¹Ģà¸Ĺ à¸Ħ +à¹Ģà¸Ĺà¸Ħ à¹Ĥà¸Ļ +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļ à¹Ĥล +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥล ย +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥลย ี +צ ר×Ļ +ĠÐļ Ñĥ +ıl ma +決 ãĤģ +ا ÙĪ +Ġ×ĵ ק×ķת +à¸Ħร ู +ĠÙħست ÙĪÙī +à¸Ľ à¹īà¸Ńà¸ĩ +à¸Ľà¹īà¸Ńà¸ĩ à¸ģัà¸Ļ +×ĵ ×ķ×ŀ×Ķ +ĠÑģ егоднÑı +س ÙĪÙĤ +ר×Ĺ ×ķ×ij +ĠØ¥ دارة +Ñħ ож +éģİ ãģİ +à¸Ħ à¸Ń +нÑĥ л +×ķ׼ ×Ķ +ÙĪ اÙģÙĤ +׼׾ ׾ +Ġ×Ķ ×ĵ×ķ +Ġl Ä©nh +Ġkh ảo +×IJ×ŀ צע +ë¨ ¸ +Ġ׼ ×Ļצ +Ġ׼×Ļצ ×ĵ +Ġдолж нÑĭ +หว ัà¸ĩ +ãĥĩ ãĤ¶ +ãĥĩãĤ¶ ãĤ¤ãĥ³ +Ġng á»Ŀ +ä¸Ń ãģ« +à¸ģลัà¸ļ มา +جÙħ اÙĦ +à¸Ķัà¸ĩ à¸ģลà¹Īาว +س ÙĥÙĨ +س ÙĨ +Ġözellik le +з еÑĢ +rz ÄĻ +×ŀ ×ķר×Ķ +Ġl ạ +×ŀ ×Ļ׳×Ļ +ר ×Ļת +ãģĿãĤĮ ãģĮ +ãģĭ ãĤĮ +ĠÙĬÙħÙĥÙĨ Ùĥ +öff entlich +г ан +ĠاÙĦØŃ ÙĦ +ĠmiÄĻd zy +ĠÑĩа ÑģÑĤи +ujÄħ cy +ĠbaÄŁ lı +ĠiliÅŁ ki +Ùģ اء +ãĥª ãĥ³ãĤ° +Ġhã ng +ĠконÑĤ ÑĢ +ĠконÑĤÑĢ ол +к оп +ש ×Ļ×¢ +ש×Ļ×¢ ×ķר +ĠÐĴ аÑĪ +Ġ×Ķ תק +ÙħÙĨ ع +ĠpolÃŃt ico +Ġг олов +ĠØ¥ ÙĬ +Ø¥ ÙĨتاج +à¸ļ ิ +Ġг овоÑĢ +ĠговоÑĢ иÑĤ +Ġph á»ķ +ĠÑģем ÑĮ +ãģ¯ ãģĤãĤĬãģ¾ãģĽãĤĵ +ĠÙĪ است +×ŀש פ×ĺ +з ем +×ŀ×ĵ ×ijר +Ġíģ ° +ĠìĿ´ ë²Ī +ê°Ģ ëĬĶ +Ġì§Ģ ìĽIJ +Ġca ÅĤy +Ġgeli ÅŁtir +Ñģк ое +pos é +Ġkh ô +à¸ķิà¸Ķ à¸ķาม +miss ão +Ġ׾ ×ŀר +Ġ׾×ŀר ×ķת +Ġb ó +à¸ķรวà¸Ī สà¸Ńà¸ļ +Ġngh á»ģ +Ġб из +Ġбиз неÑģ +ÑģÑĤ еÑĢ +ÙĪ Ùİ +楽 ãģĹãģ +楽ãģĹãģ ¿ +ãģĵãĤĮ ãģĭãĤī +wiÄħ zan +ส à¸Ńà¸Ļ +Ùħ ÙĪر +׳×ĵ ׾ +Ġ×Ķ×IJ ×ĵ×Ŀ +Ġм олод +ØŃ Ùħا +ØŃÙħا ÙĬØ© +ÑģÑĤ ÑĢан +Ġbu á»ķi +ת×Ļ ×Ļ×Ŀ +abile ceÄŁi +L Ä° +à¹Ģย à¸Ńะ +à¸Ī ร +س ÙĥاÙĨ +à¸Ļ ัà¸Ķ +Ġm ấy +ĠÐij а +s ÅĤaw +ĠÙģ ÙĦا +ĠкоÑĤоÑĢ ой +Ġпло Ñī +ĠплоÑī ад +ãĤĤ ãģĤãĤĬ +sz czÄĻ +×Ļפ ×ķ +ש×ŀ ת +owa ÅĤa +Ġn ông +צ×ij ×IJ +ĠìŀĪ ìĹĪ +ãģ¾ ãģ¨ +ãģ¾ãģ¨ ãĤģ +ÙĤÙĪ ات +ãģ¿ ãĤĵãģª +Ġ׼ ×ŀ×¢×ĺ +Ġx úc +ï¼ Ĩ +r ÄĻ +rÄĻ cz +×ĵ ×ŀ×Ļ +Ġt áºŃn +à¸Ķ วà¸ĩ +ê²½ ìłľ +п ÑĥÑĤ +Ø£ ربع +Ġ×ŀ שת×ŀש +ãĤ¿ãĤ¤ ãĥĹ +Ġìłľ ê°Ģ +Ġ׾ ׼ף +ĠобÑĢаз ом +ÙĬÙĥ ا +w ÅĤ +wÅĤ asn +ĠاÙĦÙĪØ·ÙĨ ÙĬØ© +بÙĬ ب +×ŀ ׾×Ļ +к ÑĢаÑĤ +기 ìĹIJ +ÙĤ اد +ĠÙĦ دÙī +à¸Ħวาม รูà¹ī +×ŀ×ĵ×Ļ׳ ×Ļ×ķת +ê² ¨ +Ġíĺ Ħìŀ¬ +ש ת×Ļ +м ол +Ġmá i +à¸ŀิ ม +à¸ŀิม à¸ŀ +à¸ŀิมà¸ŀ à¹Į +หล วà¸ĩ +Ġx uyên +×Ĺ ×¡×¨ +رÙĪ ÙĨ +ãģĿãģĨ ãģĦãģĨ +ãģĿãĤĮ ãģŀ +ãģĿãĤĮãģŀ ãĤĮ +Ġ׼ ש×Ķ +ÐŁ ÑĢав +×ŀ×ij צע +ع رب +Ġbü yü +פ×Ļת ×ķ×Ĺ +à¸Ī à¸ļ +ĠØ£ Ùĥبر +שר ת +×ŀ׼ ש×Ļר +ĠÙĪ Ùħع +ãģ® ãģŁãĤģãģ« +à¸Ļ ัà¸ļ +ì° ° +ãĥª ãĥķãĤ© +ãĥªãĥķãĤ© ãĥ¼ãĥł +Ġc Æ°á»Ŀng +ĠìłĢ íĿ¬ +ÙħÙĨظ ÙħØ© +Ġhiç bir +ãģ§ãģ¯ ãģĤãĤĬãģ¾ãģĽãĤĵ +ร à¸Ńย +ëIJľ ëĭ¤ +ãģĻãģIJ ãģ« +к ла +Ġürün ler +Ġki á»ĥu +ĠëĤĺ ëĬĶ +ÑĤ ки +Ñģ им +Ġchá»ī nh +ãĤĤ ãģªãģĦ +ศ รี +æĽ¿ ãģĪ +ta ÅŁ +Ġب ÙĥÙĦ +Ġ×ķ ×Ļש +vis ão +ä¼ Ŀ +ä¼Ŀ ãģĪ +ÙĦ د +׾ ×Ļ×ŀ +׾×Ļ×ŀ ×ķ×ĵ +t ória +د Ùij +اÙħ ر +Ġê·¸ëłĩ ê²Į +Ġmateria ÅĤ +à¸Ĺ รา +à¸Ĺรา à¸ļ +ãģ®æĸ¹ ãģĮ +ãģ¦ ãģįãģŁ +ض غ +ضغ Ø· +ĠÙĬ عÙĨÙĬ +ел о +×IJ×Ķ ×ij×Ķ +×¢ ×ŀ +ÅŁ ık +ìŀIJ ëĬĶ +ãĤ¿ ãĥ³ +Ġb áºŃt +×ŀשפ ×Ĺ×Ķ +к ÑĢи +б ли +สั à¸ķ +สัà¸ķ วà¹Į +ĠسÙĨ ÙĪات +ĠPh Æ°Æ¡ng +ãģ¦ãģĹãģ¾ ãģ£ãģŁ +ãģª ãģľ +Ġ×ij×IJ ×ķ +Ġc án +س جÙĦ +Ġl ẽ +ãĤ± ãĥ¼ãĤ¹ +Ġק ×Ļ×ij׾ +à¸ļà¸Ĺ à¸Ħวาม +Ġ×ķ ׼ף +ĠпÑĢедÑģÑĤав лен +Ġn á»iji +Ġcoment ário +ени ем +Ġtá» ı +l Ãł +Ġש×Ķ ×Ļ×Ķ +Ñģл ав +ĠاÙĦ ÙĪÙĦا +ĠاÙĦÙĪÙĦا ÙĬات +ÙĦج ÙĨØ© +ק×ķר ×IJ +бÑĭ ÑĤ +Ġì ¦ +Ġì¦ ī +ãģ§ãģĻ ãģĹ +หรืà¸Ń à¹Ħมà¹Ī +за ÑīиÑĤ +ÙģÙĦ سطÙĬÙĨ +Ġmi á»ħn +à¹Ģย à¹ĩà¸Ļ +ĠçalÄ±ÅŁ an +×Ļ×Ĵ ×Ķ +ĠE ÄŁ +ĠEÄŁ itim +ãĥĥãĤ· ãĥ¥ +Ġоп Ñĭ +ĠопÑĭ ÑĤ +ر غ +رغ ب +ĠÑģво иÑħ +à¸Ľà¸£à¸° à¸ķ +à¸Ľà¸£à¸°à¸ķ ู +Ġ×ŀ×IJ ×ĵ +׼ ×ķ׳×Ļ×Ŀ +à¸Ļ ี +ĠвÑĭ Ñħод +ãģ®ä¸Ń ãģ« +פ ׾×IJ +ĠÙĪ ÙĦÙĬس +פ×ķר ס +פ×ķרס ×Ŀ +Ùħ سÙĦÙħ +Ġng ôi +×ĵ ×ŀ×ķת +ãĤĴ使 ãģ£ãģ¦ +ĠпомоÑī ÑĮÑİ +Ø£ سر +бл ок +ÙĤ Ùĩ +ãģĹãģ¾ ãģĦ +ãģ¨ ãģĹãģŁ +Ġп еÑģ +ãĥī ãĥ« +×Ĺ ×Ŀ +ãģĹãģª ãģĮãĤī +ĠÐŁ ÑĢед +ãĥģãĤ§ ãĥĥãĤ¯ +å¼· ãģĦ +ש ×Ļר×ķת +д аеÑĤ +×Ļ×ij ×ķ +Ġgen ç +ил аÑģ +илаÑģ ÑĮ +ĠبÙĦ د +æĤ ª +æĤª ãģĦ +Ġ×ŀ שת +æ§ĺ ãĢħ +æ§ĺãĢħ ãģª +à¸ĺรรม à¸Ĭาà¸ķิ +ĠÙĥ اÙħÙĦ +ĠاÙĦس Ùħ +×ij×ĺ ×Ļ×Ĺ +c á +g ência +ãĤ¹ãĤ¿ ãĥ¼ +à¸Ĺำ à¸ģาร +×Ļ׾ ת +Ġ×Ļ ×ķצ×IJ +w ój +à¸ļุ à¸Ħ +à¸ļุà¸Ħ à¸Ħล +ع تÙħ +عتÙħ د +ãģĿãĤĮ ãģ« +ĠاÙĦت ارÙĬØ® +ÙĤر اء +Ġyönet im +ק שר +ĠÑģп оÑĢÑĤ +Ġר×IJש ×ķף +Ġseñ al +Ġch ắn +çĦ¡ ãģĦ +ĠдоÑģÑĤ аÑĤ +ĠдоÑģÑĤаÑĤ оÑĩно +Ġá gua +à¸ģร à¸ĵ +à¸ģรà¸ĵ ี +Ġ×ŀש ×ķ +Ġtr ải +ë² Į +ujÄħ cych +Ùģر د +à¹ĥ à¸ģล +à¹ĥà¸ģล à¹ī +ãĤĭ ãģ®ãģ¯ +ר×ķ ×ķ×Ĺ +ÙĨ Ùĥ +ĠاÙĦÙĨ ÙĤ +ãģ®ãģ§ ãģĹãĤĩãģĨ +ãģ®ãģ§ãģĹãĤĩãģĨ ãģĭ +Ùħ عرÙģ +ÙħعرÙģ Ø© +ÑĥÑī е +Ġ×ij×¢ ×Ļקר +ت صÙĦ +Ġ×Ķ×IJ ר +Ġ×Ķ×IJר ×¥ +ĠÅŀ i +à¸Ĥา à¸Ķ +íŀ ĺ +ãģªãĤĵ ãģ¨ +ĠìĤ¬ëŀ ij +l Ã¼ÄŁÃ¼ +ب اء +ĠاÙĦØ¢ خر +Ġfam ÃŃlia +ĠTh áng +Ñī ениÑı +ãĤ¯ ãĥŃ +ĠTh ứ +æĽ¸ ãģį +ен ной +ìŀ ¡ +бл аг +благ о +п ов +à¹ģ ว +à¸ĩ à¸Ħà¹Į +à¸Ńัà¸Ļ à¸Ķัà¸ļ +ãģĤ ãģĴ +ร à¹īาย +ün ün +Ġ×Ļ׼×ķ׾ ×Ķ +з он +ĠÐľ и +маÑĤ еÑĢиал +Ġë³´ ë©´ +ØŃÙģ ظ +ê Ìģ +ãģ« ãģĻãĤĭ +Ġת ×IJ +Ġ×Ķס ×ķ +ĠÑģÑĤ оÑĢ +ĠÑģÑĤоÑĢ он +ãĥĪ ãĥĥãĥĹ +ÅĤo ÅĽÄĩ +ëħ ¼ +ëĵ Ŀ +ĠÙĪاÙĦ ع +ì¶ Ķ +Ġ×Ļצ ×IJ +ĠÑĢаз дел +алÑĮ наÑı +×IJ׳ ש×Ļ +spo ÅĤ +spoÅĤ ec +spoÅĤec zn +Ø¥ عÙĦ +إعÙĦ اÙĨ +ÙĤÙĪ Ùī +íķĺë©´ ìĦľ +تط ÙĪر +Ġsi êu +Ỽ t +д ви +дви ж +Ġqu ần +k ıl +ĠпÑĢи зна +ĠH ã +ĠHã y +ĠباÙĦ ت +man ın +ãĤ« ãĥ« +Ġk á»· +ק ׾×Ļ +ëIJĺ ì§Ģ +تعÙĦ Ùħ +ìĭľ ìĦ¤ +ìĭ ¶ +íĺ ¼ +Ùĥ ÙĬÙģ +売 ãĤĬ +วิ à¸Ĭา +б ал +ĠØ£ ØŃ +Ġдолж ен +รา à¸ĩ +ราà¸ĩ วั +ราà¸ĩวั ล +Ùħ اء +ج ار +Å ļ +Ġ×ŀ×IJ ×ĸ +ר ×ŀ×Ķ +ãģĭãĤĤãģĹãĤĮ ãģªãģĦ +ét ude +czÄħ c +Ġg ór +×ł×¡ ×Ķ +Ùħ ÙĬد +ĠÐŁ еÑĢе +Ø£ خر +ãģĿãģ® å¾Į +à¹Ģà¸Ķียว à¸ģัà¸Ļ +×ŀ ×Ĵ×ķ +×ŀ×Ĵ×ķ ×ķף +д ов +mas ına +×¢ ׳×Ķ +ãĤ± ãĥĥãĥĪ +ס ×¢ +סע ×Ļ×£ +ĠT Æ° +Ġt óc +íĻľ ëıĻ +ĠÐŀ д +ĠÐŀд нако +Ġdol ayı +ؤ Ùĥد +ê³Ħ íļį +׾ ר +в еÑĩ +Ġkh ợi +Ġth ủy +×ĵ ף +ร à¸ģ +à¸ļั à¸ķร +à¹Ģà¸ģ à¹Īา +ĠاÙĦØ« اÙĦ +ĠاÙĦثاÙĦ Ø« +Ġpod rá +ער ×Ļ +ÙĨج اØŃ +Ġkh ắc +ì¸ ¡ +Ä° M +ãĤ» ãĥĥãĥĪ +ż enia +Ġ׾×Ĺ ×ijר +er Ãł +ì ´Ī +Ġkü ç +Ġküç ük +ات ÙĩÙħ +à¸ĭ à¹Į +Ùħشار ÙĥØ© +ĠاÙĦ بط +Ġd ây +ен нÑĭм +à¸Ĺีà¹Ī à¹Ħมà¹Ī +ÙĤ Ùİ +Ġv ượt +Ġtr ì +Ġwp ÅĤyw +A Åŀ +з о +ĠاÙĦس ÙĬد +à¸Ĺะ à¹Ģล +ĠÑģодеÑĢж а +ع Ø·ÙĬ +ĠاÙĦع ÙĨ +èĢħ ãģĮ +à¹Ģ หà¸Ļ +à¹Ģหà¸Ļ ืà¸Ń +Ġb ÃŃ +Ġüzer inden +ĠV Å© +Ġnu ôi +ÙĨ Ùħ +алÑĮ ного +×¢ ×Ļף +ØŃ Ø¶Ø± +ĠоÑĤ дел +ëª ĩ +ìķ ¡ +ĠÙĦدÙĬ Ùĩ +ìĻ ľ +Ġse ktör +Ġвозмож но +ĠÐĶ ж +Ġh ô +äºĭ ãģĮ +иÑĢов ание +алÑĮ ной +Ġ미 êµŃ +ر ØŃÙĦ +ĠÑįк Ñģ +пÑĢав лÑı +Ġnh á»Ŀ +ĠÄij ẩ +ĠÄijẩ y +Ùģ Ùĥر +ĠÙĪØ£ ضاÙģ +ãĥIJ ãĤ¹ +ת×ķ׼ ׳×Ļת +ÑĤел ей +ĠØ¥ÙĦÙĬ Ùĩ +ãģ¨è¨Ģ ãģ£ãģ¦ +Ġдв е +Ġch ấp +ĠL ö +à¸Ħล ิ +à¸Ħลิ à¸Ľ +Ġس ÙĪر +ĠسÙĪر ÙĬا +×ŀ×Ĺ ×ķ +st ä +д об +Ġni á»ĩm +ãģ® 大 +פר×ķ ×Ļק +פר×ķ×Ļק ×ĺ +ĠCh âu +Ġ×ŀ×Ķ ×Ŀ +Ñģк им +ĠполÑĥÑĩ иÑĤÑĮ +ÙĬ ÙĪÙħ +Ø« ÙĪر +פ×ķ׾ ×Ļ×ĺ +פ×ķ׾×Ļ×ĺ ×Ļ +ĠмеÑģÑı ÑĨ +åħ¨ ãģ¦ +ĠاÙĦÙħ جÙĦس +ĠاÙĦت اÙĦÙĬ +Ġ׊ר +åIJij ãģij +׼ ×ŀ×Ķ +б ед +Ø£ عض +أعض اء +ÙĪÙĦ د +วà¹Īา à¸Īะ +Ġb ánh +à¸Ļิ ย +à¸Ļิย ม +à¸Ľà¸£à¸° à¸ģัà¸Ļ +ÑģÑĤав иÑĤÑĮ +à¸ŀ à¸Ļัà¸Ļ +ĠÑį ÑĦÑĦ +ĠÑįÑĦÑĦ екÑĤив +Ġав ÑĤоÑĢ +ĠÄIJ Äĥng +Ġth Æ°á»Łng +ãĤĴ æĦŁãģĺ +à¸ģัà¸ļ à¸ģาร +å¾Į ãģ« +Ġya ÄŁ +ست اÙĨ +Ġli á»ģn +ãģĦ ãģ¾ +i êu +à¹Ĥà¸Ķ à¸Ļ +ĠÙĦ Ø°ÙĦÙĥ +à¹Ĥรà¸ĩ à¹Ģรียà¸Ļ +צ ×Ļ×Ĵ +ĠاÙĦÙħ عÙĦÙĪÙħات +ç§ģ ãģŁãģ¡ +à¸Ĺีà¹Ī à¸Ħุà¸ĵ +ãģ«ãģª ãģ£ãģ¦ãģĦãĤĭ +×ŀ×ĵ ×Ļ׳×Ķ +ס ׼×Ŀ +Ġв не +à¸ŀ à¸Ļัà¸ģà¸ĩาà¸Ļ +ÑĢ ей +à¹Ģà¸Īà¹īา หà¸Ļà¹īาà¸Ĺีà¹Ī +ĠHi á»ĩn +Ġméd ico +ĠتØŃ ÙĤÙĬÙĤ +ÑĮ ÑĤе +miÅŁ ti +ÙĤÙĬ ادة +ãĤı ãģĭãĤĬ +มา à¸Īาà¸ģ +ëħ Ģ +ãģ«éĸ¢ ãģĻãĤĭ +×IJר×Ĵ ×ķף +m ètre +Ġעצ ×ŀ×Ļ +ĠCh úa +รูà¹ī à¸Ī +รูà¹īà¸Ī ัà¸ģ +ì£ Ħ +ëĭ µ +à¹ģà¸Ĺ à¹ī +Ġgeç en +Ġlan ça +ĠاÙĦ بØŃØ« +×ĵ ×ŀ×ķ +ãģ¯ ãģĺ +ãģ¯ãģĺ ãĤģ +Ġdön Ã¼ÅŁ +è¿ij ãģı +à¹Ģส ม +à¹Ģสม à¸Ń +ëĿ ½ +Ġü ç +á» ŀ +ÑĪ аÑı +à¸Ĺ ร +ØŃ ÙĤÙĬÙĤØ© +à¸Ĥà¸Ńà¸ĩ à¸ģาร +Ġ무 ìĹĩ +Ġ×Ķ ׼ר +ĠاÙĦص ÙĬÙĨ +ĠлÑİ Ð´Ð¸ +à¸ķ าย +ب ÙĪÙĦ +Ġvi êm +Ġthi á»ĩu +à¸ģ à¸Ķ +Ġ׾ ×ĵ×ijר +פ ׳×Ķ +×IJר ×ij×¢ +س Ùī +ĠاÙĦسÙĬ اس +ĠاÙĦسÙĬاس ÙĬØ© +yd ı +ÙĪØŃد Ø© +ĠдеÑıÑĤелÑĮ ноÑģÑĤи +Ġ×ķ×Ķ ×ŀ +п еÑĩ +пеÑĩ аÑĤ +иÑĢов аниÑı +ĠÑģ ог +ĠÑģог лаÑģ +Ġ׼ ×ĵ +Ġ׼×ĵ ×IJ×Ļ +ĠиÑģполÑĮзов аÑĤÑĮ +ס פ×ķר×ĺ +Ġil çe +exp érience +ĠTh á»Ŀi +Ä° K +à¹Ħà¸Ł à¸Łà¹īา +ëĵ¤ ìĹIJê²Į +à¸Ľà¸£à¸° à¹Ģà¸ł +à¸Ľà¸£à¸°à¹Ģà¸ł à¸Ĺ +Ġmü mk +Ġmümk ün +Ġ×IJ×ķת ׳×ķ +ìĦ± ìĿĦ +ĠìĿ´ ìľł +زÙĬ ارة +Ġolduk ça +r ób +ĠØ£ ÙĨا +Ġ×Ķ ×ij×Ļ +Ñģ ен +×¢ ×Ļקר +×Ļ×ĵ ×ķ×¢ +d zÄħ +Ùħ عÙĦÙĪÙħات +Ø´ اب +Ġpar ça +à¸Ļะ à¸Ħะ +ب اس +ĠÑĤоÑĢ г +ĠÑĤоÑĢг ов +Ġ×Ĺ ×ĵר +׼ ר×ĺ +׼ר×ĺ ×Ļס +ĠA yrıca +ÃªÌ £ +ìľ ¨ +ĠÑĤак ие +Ġ×ŀצ ×ķ×Ļ +ãĥ©ãĥ³ ãĤŃãĥ³ãĤ° +ש×Ļ×ķ ×ķק +åīį ãģ® +ĠB ảo +Ñī Ñĥ +æĹ© ãģı +ĠPh òng +à¸ŀระ ราà¸Ĭ +פ ×Ĺ×ķת +Ġг л +Ġгл аз +à¸Ĺ à¹Īา +Ġd ạy +ÑĢ оÑģÑĤ +à¹Ĥà¸Ķย à¹Ģà¸īà¸ŀาะ +Ġqu áºŃn +Ġ×Ĺ×ijר ×ķת +m ême +mÄ±ÅŁ tı +ĠاÙĦت داÙĪÙĦ +Ġn ạn +Ġ×Ķ ×ĵ×Ļ +ĠاÙĦØ· رÙĬÙĤ +×Ĵ ×ķת +Ġ×Ķ ×ĵר×ļ +ujÄħ ce +Ġch ữ +ãĤĤãģ® ãģ® +ë° Ľ +ãģķãĤĵ ãģ¯ +Ġyard ım +ĠاÙĦع Ùħ +Ġì§Ħ íĸī +Ġ×Ļ ×Ĺ +Ġ×Ļ×Ĺ ×¡×Ļ +ĠاÙĦÙħ دÙĬÙĨØ© +Ġc ú +à¸ģี ฬ +à¸ģีฬ า +Ġni ên +mis ión +׳×Ļס ×Ļ +׳×Ļס×Ļ ×ķף +Ġвоз ÑĢаÑģÑĤ +Ġ×¢×ķש ×Ķ +ĠÙħ دÙĬر +Ñı ÑģÑĮ +ØŃ Ø¬Ùħ +íĻĺ ê²½ +ĠاÙĦØ£ خرÙī +u ÃŁer +ĠاÙĦعاÙĦÙħ ÙĬØ© +ĠNg á»įc +êµIJ íļĮ +ä¸Ĭ ãģ§ +×Ļ×Ķ ×ķ×ĵ +×Ļ×Ķ×ķ×ĵ ×Ļ×Ŀ +Ùħس اعدة +Ġжиз нÑĮ +ĠпоÑĤ омÑĥ +ĠاÙĦÙħ ÙħÙĦ +ĠاÙĦÙħÙħÙĦ ÙĥØ© +ĠG ör +ر ÙIJ +×ŀק ×ķ×ŀ×ķת +åĩºæĿ¥ ãĤĭ +ÑĦ ÑĤ +ĠìĿ´ ìłľ +ĠÑĢ ем +ĠÑĢем онÑĤ +ת ×ķ×ļ +æĻĤ ãģ¯ +ãĤīãĤĮ ãģªãģĦ +alt ı +家 ãģ® +ĠاÙĦØ¥ عÙĦاÙħ +리 ëĬĶ +ãģĭãĤī ãģ¯ +ĠH ạ +ãģĤ ãģ® +×ĵ×Ļ ×ķף +رÙĬ س +Ġsoci etÃł +ĠاÙĦÙĥ بÙĬر +Ġ×ij ×ŀס +Ġ×ij×ŀס ×Ĵר +Ġ×ij×ŀס×Ĵר ת +ĠìŀĪ ìľ¼ë©° +Ġn ặng +Ùĩ Ùī +ĠB Ãł +×ŀר ×ķ +Ġj ÄĻ +ĠjÄĻ zy +ĠjÄĻzy k +Ġ׼ ×ŀ×ķ×ijף +×¢ ׾×Ķ +à¸Ĺีà¹Ī à¹Ħà¸Ķà¹ī +ãģ¾ ãģĹãĤĩãģĨ +×ŀס פר +Т Ðŀ +سÙĬاس Ø© +Ġкажд Ñĭй +ë² ł +t ım +y á»ĩn +ร ีà¹Ī +ĠдеÑĤ Ñģк +วิà¸ĺี à¸ģาร +m ówi +×ĺ×¢ ×Ŀ +×Ķצ׾ ×Ĺ×Ķ +ض ÙĬÙģ +ĠÑħоÑĤ Ñı +ãĤĵãģ§ ãģĦãĤĭ +à¸Ħา à¸Ķ +à¸Ħร à¸ļ +Ġк ÑĥÑĢÑģ +ĠbaÅŁ arı +×ijר ×ķ +ÙĬع Ø© +ĠÐĿ Ñĥ +à¸Ħวาม à¹Ģà¸Ľà¹ĩà¸Ļ +Ġ׾ ×ŀש׾ +Ġì¢ĭ ìĿĢ +Ùħؤس س +Ùħؤسس ات +Ġpréc is +Ġth ảo +à¸ģà¹ĩ à¸Ħืà¸Ń +Ġש ׼׾ +führ ung +ãģĦ ãģ§ +à¹ģละ มี +à¸ģà¹ĩ มี +Ġש ש +м ел +Ġкни г +ĠباÙĦ ÙĨ +ĠباÙĦÙĨ سبة +Ġald ı +ÑĤ ай +Ġ×Ĺ×ĵ ש×Ļ×Ŀ +å®Ł ãģ¯ +ع ÙĪا +ĠìĿĺ 미 +из м +ÑĢабоÑĤ аÑĤÑĮ +Ùģ ص +Ġ×ij׳ ×ķסף +ãģ¨ãģĹãģ¦ ãĤĤ +à¹Ģà¸Ľà¹ĩà¸Ļ à¸Ĺีà¹Ī +ĠÑģлед ÑĥеÑĤ +èĢĥãģĪ ãģ¦ +Ġ׼ ×Ļ×ķ×Ŀ +ÑģÑĤ Ñĭ +׼׾׼ ׾×Ļ +æµģ ãĤĮ +ãĤĴ ãģ¤ãģij +Ñĩ аÑĤ +×Ļ׼ ×ķף +×Ļר ×Ļ +ları yla +ãĤ¤ ãĥ¡ +ãĤ¤ãĥ¡ ãĥ¼ãĤ¸ +׳×ĸ ק +Ġci ò +Ġs ın +Ġsın ır +à¸Ļ à¸Ħร +к аÑĤ +Ġl á»Ĺi +ëŀ Į +تÙģ اص +تÙģاص ÙĬÙĦ +ëĨ ĵ +ĠÙħ ض +il miÅŁ +بار Ùĥ +ÐĿ Ðĺ +Ġth ẩm +Ġ×IJ×ķת ×ļ +ĠпÑĢин им +ĠпÑĢиним а +Ġyö nt +Ġyönt em +Ġ×ŀק ×ij׾ +Ġktó rego +ê· Ģ +شر Ùģ +د اÙħ +ãģĦãĤį ãģĦãĤį +ĠAl ém +Ġgör ü +Ġgörü nt +Ġgörünt ü +د س +ÑĪ ки +г ÑĢад +Ġl ạc +Ġs ữa +ãĤīãĤĮ ãģ¾ãģĻ +o Ãłi +Ñī ен +ãģĭ ãģªãģĦ +Ġп оп +Ġпоп Ñĥ +ĠпопÑĥ лÑıÑĢ +ĠاÙĦÙħ ÙĪÙĤع +rä g +ï¼ ¡ +íķ Ħ +ãĤĴè¦ĭ ãĤĭ +اÙħ ا +ĠاÙĦØŃ Ø±Ø¨ +ĠÐŁ а +Ġ׾ ×IJתר +Ġt á»ijc +×ij ׾×Ķ +ر ئÙĬس +в Ñĥ +ÙĬ دÙĬ +каз ан +Ġ׊ש×ij×ķף +h ôtel +×¢ ×ķ׳×Ķ +ب ÙĨÙĬ +×ŀ ×ķ׾ +Ġд нÑı +éĽ£ ãģĹãģĦ +вед ениÑı +Ġ×ķ ×ŀת +н апÑĢимеÑĢ +ÙĤ ابÙĦ +Ġrésult at +ĠÑĢазвиÑĤ иÑı +ر Ùij +ìłĦ 문 +ĠاÙĦÙħ زÙĬد +ĠìľĦ íķ´ìĦľ +ëĨ į +íĻ ķ +ĠThi ết +íĮ ¨ +malı dır +Ġcz ÅĤ +ĠczÅĤ owie +ĠczÅĤowie k +ĠÙĦ بÙĨ +ĠÙĦبÙĨ اÙĨ +üs ü +ãģªãĤĵ ãģł +Ġżyc ie +ĠÑħоÑĢоÑĪ о +æĸ¹ ãģ« +ëĭ¤ ë©´ +иÑĩеÑģ каÑı +ער ×Ļ׼ +ער×Ļ׼ ת +ãģ¾ãģĽãĤĵ ãģ§ãģĹãģŁ +ĠÑģоб ой +Ġg á»Ĺ +Ġдел аÑĤÑĮ +da Äĩ +аÑĢ а +róż ni +à¹Ģล ีà¹ī +à¹Ģลีà¹ī ย +à¹Ģลีà¹īย à¸ĩ +à¸Ŀ าà¸ģ +Ġت ÙĤ +ĠتÙĤ دÙĬ +ĠتÙĤدÙĬ Ùħ +หà¸Ļ ุà¹Īม +Ġmü cade +Ġmücade le +ì§Ģ 를 +ãĤ¤ ãĤ¹ +ĠØ£ ساس +jÄħce go +ĠÅŁ eh +н ÑĤеÑĢ +ÑĨи Ñİ +ï» » +ÑİÑī его +à¹Ĥà¸Ľà¸£ à¹ģ +à¹Ĥà¸Ľà¸£à¹ģ à¸ģรม +Ġmie Äĩ +ØŃÙĥÙĪÙħ Ø© +ãģ§ãģĹãģŁ ãģĮ +×Ļס ×Ķ +ãĤĤãģ® ãĤĴ +Ġ×ŀ ×IJת +สุà¸Ķ à¸Ĺà¹īาย +Ġc Å© +ÙĨ سب +ĠпÑĢ оÑĩ +Ġд ней +ĠÑįÑĤи Ñħ +׾ ×ŀת +нÑı Ñı +Ñį к +Ġì§Ģ ëĤľ +มหา วิà¸Ĺยา +มหาวิà¸Ĺยา ล +มหาวิà¸Ĺยาล ัย +d ão +ĠMá y +ĠêµŃ ê°Ģ +à¸ļุ รี +×Ĵ ×Ļ׾ +ĠÑĤÑĭ ÑģÑı +ĠÑĤÑĭÑģÑı Ñĩ +Ùģ Ùĥ +ĠÐĺ Ñģ +è¡Į ãĤıãĤĮ +פר ×ĵ +ãģ¤ ãģį +à¸Ħร à¸Ńà¸ļ +à¸Ħรà¸Ńà¸ļ à¸Ħรัว +à¸Ĥึà¹īà¸Ļ มา +ä»ĬæĹ¥ ãģ¯ +ĠìĤ¬ëŀĮ ìĿ´ +עצ ×ŀ×Ķ +п оÑĢ +ĠK ỳ +Ġ Æ¡n +Ġth Äĥm +Ùģ اÙĤ +ãģļ ãģ« +Ġ׾ קר +Ġ׾קר ×ķ×IJ +اÙģ ÙĬØ© +Ùħ Ùİا +г аÑĢ +ص ÙĦا +صÙĦا Ø© +Ġ×ŀ ×ĸ×Ķ +lı ģını +Ġ×IJ ×Ļ׳×Ķ +к ÑĢо +Ġng Æ°Æ¡i +Ġв ним +Ġвним ание +jÄħ cy +ÙĢÙĢÙĢÙĢ ÙĢ +Ñģ Ñħод +ãģªãĤĵ ãģĭ +×ŀ ×Ļ׾ +Ġ×Ķ×IJ ×Ĺ +ãĤı ãģªãģĦ +ع سÙĥر +ĠìĦ¸ ê³Ħ +ĠÑĩ его +ĠÑģÑĢед ÑģÑĤва +ĠÐł аÑģ +ãģª ãģģ +ÙĨ Ùģس +ר×Ļ ×ķף +Ñģ Ñĥд +ĠìĿ¸ ê°Ħ +ĠاÙĦÙħ ÙĤبÙĦ +ÙĨ عÙħ +تÙĪ Ùģر +ש ×ij×¢ +ı lm +ılm Ä±ÅŁ +Ġ×ľ×ª ת +تص Ùģ +×Ķפ ×ķ×ļ +à¹ĥà¸Ļ à¸Ľà¸µ +ìĿ´ ê³ł +Ùģ ÙĪز +à¸ľà¸¥ à¸ĩาà¸Ļ +ĠGi áo +à¸ļà¸Ńà¸ģ วà¹Īา +Ġd Ä±ÅŁ +ĠdÄ±ÅŁ ında +ì£ ½ +Ġdzie ÅĦ +к ÑĨии +и ÑĨе +ãģ® ä¸Ģ +ع Ø´ +пÑĢ еÑģÑģ +หà¸Ļ à¹Īà¸Ńย +ลัà¸ģษ à¸ĵะ +Ġpossibilit Ãł +à¹Ħà¸Ķà¹īรัà¸ļ à¸ģาร +หย ุà¸Ķ +Ġphi ên +çĶŁ ãģ¾ãĤĮ +Ø· ÙĪÙĦ +ÑĦ ин +f ür +ØŃ ÙĬاة +íĸ ĪìĬµëĭĪëĭ¤ +׼ ׳×ķת +à¸Ľà¸£à¸° ส +à¸Ľà¸£à¸°à¸ª à¸ļ +à¸Ľà¸£à¸°à¸ªà¸ļ à¸ģารà¸ĵà¹Į +ëIJĺ ìĹĪ +Ġkaż dy +Ġl uyá»ĩn +ĠоÑĢганиз аÑĨии +å°ij ãģªãģı +ÑģÑĤÑĢо ен +Ġtécn ico +ק ×Ķ׾ +Ġ×ķ×IJ ×Ĺ +ĠعÙĦÙĬ Ùĥ +Ñī ение +Ġ×Ķ ×Ļ׾×ĵ×Ļ×Ŀ +ÙĪس ائÙĦ +Ġ×ķ ×Ķת +تÙħ ÙĬز +ĠÑģ казал +Ġпол и +Ġ×Ķ×ŀ ס +ÙĦÙij Ùİ +Ùħؤس سة +Ġ×ŀ ×Ļ×ĵ +ãģ£ ãģ¡ +ĠëĦĪ 무 +à¸ŀ ี +Ġt ặng +Ġt ấn +ר ש×Ŀ +Ġméd ica +Ġ×¢ ×ķ×ŀ +Ġ×¢×ķ×ŀ ×ĵ +ÑĦ оÑĢ +Ùħر Ø© +Ġvat anda +Ġvatanda ÅŁ +Ġдел о +à¸Ļ ม +ãģ¨ åIJĮãģĺ +Ùģ Ùī +Ñģ оÑĢ +Ġ×Ķס ר×ĺ +Ġép oca +ìłķ ì±ħ +ĠÑģвÑıз ан +ض رب +ĠÙĦ ÙĨا +Ġuży wa +ĠاÙĦج ÙĬØ´ +Ñİ ÑĢ +×ijס ×ķ×£ +Ġм Ñĥ +ĠмÑĥ зÑĭк +bilit é +Ġma ç +س Ùİ +ت ÙĦÙĥ +ãģ ¬ +ÙĬ ÙĦا +ÑĪ ла +ÙĢÙĢ ÙĢ +Ġод ной +зв ан +ĠÑģ ÑĢаз +ĠÑģÑĢаз Ñĥ +ÙĨ ظÙħ +را Ùĩ +ĠÙĦÙĩ ذا +׼ ×ķר +Ġ×Ķש ×ij×ķ×¢ +Ġ×Ķש ת +ĠQu ảng +ãĥ« ãĥ¼ +ãģĪ ãģªãģĦ +×ĺ ×IJ +Ġmi á»ģn +ĠPh áºŃt +ĠاÙĦس ÙĪÙĤ +Ä Ĥ +ĠاÙĦج Ùħع +ĠاÙĦجÙħع Ø© +ÑİÑī ей +a ÅĤem +عت ÙĤد +Ø£ ÙĦÙħ +Ñģ ке +ĠìĿ´ íķ´ +ÙĨس Ø® +è¨Ģ ãģĦ +д обав +سب ÙĤ +×¢×ķר ר +ÑĤи п +ãģĿãģĵ ãģ§ +vis ión +عÙĪد Ø© +ë¨ ¹ +×ŀ ×ĸר×Ĺ +ĠØ¥ ØŃ +Ġ׾×ij ×Ļף +Ġ׾צ ×IJת +Ġyard ı +Ġyardı mc +Ġyardımc ı +Ä° Z +ק פ×Ķ +tr é +liÄŁ ini +клÑİÑĩ а +Ġüret im +Ġa yrı +ĠkiÅŁ iler +à¸Ħ à¹īà¸Ļ +à¸Ħà¹īà¸Ļ หา +ĠS á»± +Ġ׼ ס +Ġ×Ľ×¡ ×£ +ĠÑĤак иÑħ +ĠXu ân +Ġл ег +Ġлег ко +Ø«ÙĤ اÙģØ© +ÐĿ Ðŀ +ãĤ¹ãĤ¿ ãĥĥ +ãĤ¹ãĤ¿ãĥĥ ãĥķ +åIJĪ ãģĦ +Ġ×Ķש ×Ļ×ŀ×ķש +man ız +ĠÐĴ аÑģ +g ün +ìľĦìĽIJ íļĮ +Ġwsp óln +ĠÑģв ое +í ĥģ +à¹Ģà¸Ļ ีย +ÙĪب Ø© +в Ñıз +ı dır +ëIJĺ ìĹĪëĭ¤ +ĠdeÄŁi ÅŁtir +ãĤĭ ãģĵãģ¨ãģĮ +Ġ×Ĺ×ĵ ש×Ķ +ãĤīãĤĮ ãģ¦ãģĦãĤĭ +×Ĺ×Ļ ×Ļ×ij +ĠÐļ аÑĢ +׳×Ļת ×ķ×Ĺ +Ġק×ĺ ף +ר ×ĸ +ÙĪ غ +èªŃ ãģ¿ +Ġت ÙĤÙĪÙħ +ĠÙĥ اÙĦ +à¸Ŀ ึà¸ģ +Ġë°ľ ìĥĿ +ológ ico +ر اع +à¹ģà¸ģà¹ī à¹Ħà¸Ĥ +ĠÑĢабоÑĤ Ñĥ +ÙĨÙij Ùİ +à¸Ńยูà¹Ī à¸Ĺีà¹Ī +ĠاÙĦØ« اÙĨÙĬØ© +ĠNh ân +Ñħ ваÑĤ +ö ne +Ġع دة +à¹ģ สà¸ĩ +ÑĤ оп +пÑĥÑģ ка +شر اء +ĠÐļ ом +Ġפע ×ķ׾×Ķ +ìĤ¬ ìĿ´ +ìĤ¬ìĿ´ íĬ¸ +è¡Į ãģ£ãģ¦ +Ġ×Ķ ×Ķת +ĠÑģÑĤ оÑĢо +ĠÑģÑĤоÑĢо нÑĭ +در س +à¸ĭ ู +à¸ķà¹Ī ำ +ĠØ£ بÙĬ +под об +ãģ« ãģ¦ +ار تÙģاع +ĠÙħ ؤ +ик ов +ge führt +มืà¸Ń à¸ĸืà¸Ń +ĠÙĦ ÙĤد +ĠØ£ÙĨ Ùij +سÙĬ طر +ãģ¾ãģļ ãģ¯ +ס ×ĵ +Ñģк олÑĮко +ãģ¿ãģŁãģĦ ãģª +×ĵר ×Ĵ +×¢ ×Ļ×ĵ +à¹ĥหà¹ī à¸ļริà¸ģาร +ĠÐĶ и +×ij×¢ ×Ļ×ķת +Ġ×Ķ×Ĺ ×ķ +пиÑģ ÑĮ +ĠاÙĦØ® ÙĦ +б ав +ĠÄ° lk +ĠاÙĦØ® Ùħ +ĠاÙĦØ®Ùħ ÙĬس +ĠÙĬ ÙĤÙĪÙħ +æĻĤ ãģ® +ĠsÅĤ ow +ĠØ£ ÙĩÙħ +Ø®ÙĦ ÙĤ +ĠØ£ صبØŃ +Ġchứ a +Ġth ác +Ùģ اÙĦ +Ġch á»Ŀ +ĠاÙĦØ® ار +ĠاÙĦخار ج +ĠاÙĦخارج ÙĬØ© +Ø· ائر +Ġt Ãł +ĠtÃł u +à¸ģล à¹īà¸Ńà¸ĩ +ĠاÙĦÙħر Ø£ +ĠاÙĦÙħرأ Ø© +åħ¨ ãģı +ĠÃĸ n +çļĦ ãģ«ãģ¯ +Ġpiè ce +×Ĵ ×Ļ×ij +ĠاÙĦ ÙĪاÙĤع +ä»Ĭ ãģ® +ĠاÙĦÙħ ÙĤ +cz nÄħ +Ùģع اÙĦ +ен ного +ĠÑĦак ÑĤ +ìĭł ì²Ń +ĠÐŀ ни +ĠاÙĦبÙĦ اد +ов иÑĩ +ëı Į +ÑĦ ÑĥнкÑĨи +Ġìĸ´ ëĬIJ +ãĥķãĤ© ãĥ¼ +d ÃŃ +ил оÑģÑĮ +Ùħ Ùī +ĠاÙĦØ£ÙħرÙĬ Ùĥ +ĠاÙĦØ£ÙħرÙĬÙĥ ÙĬØ© +×ĺ ×Ļפ×ķ׾ +íĶĦ ë¡ľê·¸ +íĶĦë¡ľê·¸ ëŀ¨ +Ġש ×ķ׳×ķת +Ø´ ÙħÙĦ +ĠпаÑĢ а +Ġ×Ķ×Ĺ ×ķק +ÙĪز ارة +ãģ¨ ãģĻãĤĭ +Ġqu ảng +ĠaÄŁ ır +ĠاÙĦÙĦ ج +ĠاÙĦÙĦج ÙĨØ© +ê¸ ´ +ĠT ân +ج ÙħÙĦ +д ол +à¹ģà¸ŀ à¸Ĺย +à¹ģà¸ŀà¸Ĺย à¹Į +Ġר×IJ ש×Ļ +Ñī ей +Ġçev re +Ġкомп лекÑģ +Ġ×ij ×ŀש×ļ +Ġalt ın +ĠØ£ عÙħاÙĦ +ĠÑģво его +ãĤĪ ãģĦ +×Ĺ׾ ×Ļ×ĺ +×ŀ׳ ×¢ +Ġר ×ij×Ķ +ĠØ£ÙĬضا Ùĭ +×ĸ ׾ +ĠاÙĦسÙĬ اسÙĬ +æĢĿ ãģĨ +קר ק +קרק ×¢ +ĠاÙĦÙģ رÙĬÙĤ +б иÑĤ +ק ׳×Ķ +ĠØ¥ ÙĨÙĩ +ĠÐĴ ам +Ðł Ðŀ +ãĥĪ ãĥª +å¿ħè¦ģ ãģª +Ġch âu +ç¶ļ ãģij +Ġçöz üm +gÅĤ ow +ع ÙĤÙĦ +売 ãĤĭ +i ết +à¸Ĭิ à¹īà¸Ļ +ĠØŃÙĤ ÙĪÙĤ +Ø·ÙĦ ع +ĠÄij en +ĠÙĥ اÙģØ© +ãģ® ãģĶ +Ġë ¬ +Ġë¬ ¼ +Ġ물 ë¡ł +Ġرس ÙĪÙĦ +з ам +зам ен +Ġkullan ıcı +×¢ ×ķ׾ +èī² ãĢħ +ÑĪи ÑĢ +Ġ׊ש +Ġwy gl +Ġwygl Äħda +ש ×Ļ×ŀ×ķש +å¿ĺ ãĤĮ +×¢ ×Ļצ×ķ×ij +ĠاÙĦس ÙĪرÙĬ +å°ij ãģªãģĦ +Ġпо иÑģк +สำ à¸Ļัà¸ģà¸ĩาà¸Ļ +Ġ×ŀצ ×ĵ +Ġmü ÅŁ +ĠmÃ¼ÅŁ ter +ĠmÃ¼ÅŁter i +ĠÙħÙĨ ÙĩÙħ +à¸ķำ à¹ģ +à¸ķำà¹ģ หà¸Ļ +à¸ķำà¹ģหà¸Ļ à¹Īà¸ĩ +ÅĽ mie +Ġש ×ł×ª +Ġ×Ķ פ×Ļ +פר ש +×¢×ijר ×Ļת +สà¸Ļ ัà¸ļ +สà¸Ļัà¸ļ สà¸Ļุ +สà¸Ļัà¸ļสà¸Ļุ à¸Ļ +è¨Ģ ãģ£ãģ¦ +à¸ģาร à¸Īัà¸Ķ +ĠMo że +из аÑĨии +ứ t +ĠÙĪب عد +ĠdeÄŁ ild +ĠdeÄŁild ir +Ġת ×ŀ +Ġ×ŀ×ŀ ׳×ķ +話 ãĤĴ +ĠÑĨ ена +Ġth úc +×Ļ×ŀ ×ķף +ĠB áo +ãĤĴ åıĸãĤĬ +å®ī ãģĦ +Ġ×¢×ķש ×Ļ×Ŀ +èĩªåĪĨ ãģĮ +l ée +ãĤĭ ãģ®ãģ§ +иÑĢÑĥ еÑĤ +ãģ¦ ãĤĭ +ست ر +ĠاÙĦØŃ ÙĬ +×Ļ׾ ×ķת +Ġ×Ĺ ×ij +ÙĤر Ø£ +تÙħ ÙĥÙĨ +س ائÙĦ +prü f +ãģĭ ãģijãģ¦ +ĠÑģоб ÑģÑĤвенно +ĠìľĦ íķĺìŬ +׾ ×Ļ×ĺ +ãģĮ å¤ļãģı +ÙĬت Ùĩا +ç«ĭ ãģ¦ +ม à¸Ńà¸ļ +ìĭľ ìŀ¥ +оÑĢ а +Ġs avaÅŁ +×ĺ×Ļ×ij ×Ļ +×ij ׳×ķ +Ùħا ذا +기 ê°Ħ +ãģªãģ© ãģ§ +Ġ×ŀ ת×Ĺ×Ļ׾ +Ġnhi á»ħ +Ġnhiá»ħ m +ка ÑĢ +каÑĢ ÑĤ +Ġ׾×Ķ שת×ŀש +׳ ×Ļ×Ĺ +اد ÙĬØ© +ราย à¸ĩาà¸Ļ +Ġprzy kÅĤad +Ñī ий +ØŃض ÙĪر +Ġh ôn +à Ŀ +ת ×ķצ×IJ×ķת +راب Ø· +Ġb ếp +ĠполÑĥÑĩ и +åĩºä¼ļãģĦ ç³» +à¸Ľà¸¥ à¹Īà¸Ńย +ĠاÙĦØ´ باب +اÙĩ ÙĦ +ä»Ĭ ãģ¾ãģ§ +رج ع +ãĤ¶ ãĥ¼ +ÙĤ Ùģ +ĠGro ÃŁ +ĠíļĮ ìĽIJ +اج ر +Ġ×ij×ŀ קר×Ķ +Ġseg urança +fü hl +ãģ¦ ãģĦãģı +หม à¸Ń +ĠкоÑĤоÑĢ ом +ĠN Äĥm +ĠdÅĤ ugo +ÙħÙĨ ØŃ +ש×ķ ×ķ×Ļ +ĠØ£ÙĬ اÙħ +ส à¸łà¸²à¸ŀ +r zÄħ +شر Ùĥات +ãĤĴ èĢĥãģĪ +д аÑĢ +à¸Ľà¸£à¸° à¸Ĭุม +Ġ×ķ×IJ ×ĸ +i á»ĩn +Ġt Æ°Æ¡i +ש ×Ļ×Ĺ +à¸Ń à¹Īà¸Ńà¸Ļ +æĽ¸ ãģĦãģ¦ +Ġng ữ +×ij×Ļ×ĺ ×Ĺ +×ij×Ļ×ĺ×Ĺ ×ķף +Ġs ẵ +Ġsẵ n +ì§Ģ ëıĦ +ĠпÑĢ еп +ĠпÑĢеп аÑĢаÑĤ +Ġна ÑĥÑĩ +ĠÃľ nivers +ĠÃľnivers ites +ĠÃľniversites i +Ġ×Ĵ×ĵ ×ķ׾×Ķ +Ġ×Ķ ×ł×ª +Ġ×Ķ×ł×ª ×ij×¢ +ãģ§ãģĤ ãģ£ãģŁ +Ġmies iÄħ +ĠmiesiÄħ c +г ÑĢам +гÑĢам м +Ġبش Ø£ÙĨ +ĠÑħ ÑĢ +ק ×Ļ×ĵ +ק×Ļ×ĵ ×ķ×Ŀ +Ø´ Ùĥر +Ġ á»ķ +Ġá»ķ n +ãģĮãģĤ ãģ£ãģ¦ +ãģķãĤĮ ãģ¾ãģĻ +Ġ×Ĺ ×ķ×ĵ +Ġ×Ĺ×ķ×ĵ ש×Ļ×Ŀ +ÙħÙĪا جÙĩ +ÙħÙĪاجÙĩ Ø© +أش خاص +ب غ +à¹Ģรียà¸Ļ รูà¹ī +ãģĹãģ¦ ãģĦãģı +Ġs ạn +å¿ħ ãģļ +׳ ×Ļ×Ĵ +׳×Ļ×Ĵ ×ķ×ĵ +باÙĦ غ +׊ש×ŀ +×Ĺש×ŀ ׾ +Ġnap raw +Ġnapraw dÄĻ +Ø´Ùĩ اد +×IJ ×ķ×Ķ +×IJ×ķ×Ķ ×ij +и ÑĨÑĭ +Ġ×Ķ ר׼×ij +ëŀ ij +Ġת ×¢ +Ġ×Ķ ×Ļש +Ġ×Ķ×Ļש ר×IJ +Ġ×Ķ×Ļשר×IJ ׾×Ļ +Ø£ ÙħÙĨ +ÑİÑī аÑı +sk ór +LER Ä° +Ġ×Ķ×IJ×Ĺר ×ķף +×¢ ׳ק +ĠÙĪ ÙĥÙĦ +ãģĵãģĵ ãģ§ +Ġqu án +liÄŁ in +à¸ģà¸İ หมาย +Ø· Ùħ +Ø£ جÙĩ +أجÙĩ زة +ĠEr doÄŁan +ãģ§ ãģĬ +Ġв ÑĢа +ĠвÑĢа Ñĩ +ĠPh ó +à¸Ĭั à¹Īว +à¸Ĭัà¹Īว à¹Ĥม +à¸Ĭัà¹Īวà¹Ĥม à¸ĩ +Ġph úc +×Ļפ ×ķת +×¢×Ļ ×ķף +Ġduż o +ãĥģ ãĥ¼ãĥł +ĠÙĬ Ùİ +Ġзад аÑĩ +Ġ×Ĵ×ij×ķ×Ķ ×Ķ +Ġ׼ ׼׾ +лож ен +ét at +Ġng Äĥn +èµ· ãģį +ĠTi ến +ص عب +Ġexperi ência +Ø® Ùħ +à¸ģาร à¸Ĺำà¸ĩาà¸Ļ +س ÙĬد +ĠD á»± +ĠкоÑĤоÑĢ ого +lad ıģı +Ġkh á»ķ +Ġê³Ħ ìĨį +Ñī ик +สà¹Īวà¸Ļ à¸ķัว +з оÑĢ +ÙĨ Ùı +Ġ à¸Ķัà¸ĩ +Ġà¸Ķัà¸ĩ à¸Ļัà¹īà¸Ļ +Ġc ấu +ĠÄij á»ijc +о ÑĦ +ĠاÙĦØ£ عÙħاÙĦ +ãģªãģı ãģ¦ãĤĤ +×ķ׼ ×Ļ×Ŀ +à¹ģ à¸Ľ +ĠB ên +ãĥ¯ ãĥ³ +Ġgi ám +ĠÅŀ u +Ġd áng +ع ÙĦÙĬ +à¹Ģà¸ģ ษ +à¹Ģà¸ģษ à¸ķร +ÙĪج ب +н нÑĭе +ÙĤ ضاء +à¸Ħว à¸ļ +à¸Ħวà¸ļ à¸Ħุ +à¸Ħวà¸ļà¸Ħุ ม +ãģ¤ ãģ¤ +ĠVi á»ĩc +×ŀ×ij ×ĺ +ש×Ļת ×ķ×£ +Ġв едÑĮ +k aza +kaza ÅĤ +à¸ķำ รวà¸Ī +ãĤ¿ ãĥ« +Ġпов Ñĭ +ĠповÑĭ ÑĪен +ĠS ợ +ĠìĦ¤ ëªħ +ĠÃĩ ünkü +ìĥĿ íĻľ +Ö ¾ +ãĤĮ ãģ¦ãģĦãĤĭ +Ġ×ij ר×IJש +ר ×ķ×Ĵ +Ġо ÑĦи +ĠоÑĦи ÑĨиалÑĮн +ĠÑĥ ÑģÑĤанов +ĠÑĥÑģÑĤанов лен +ĠاÙĦÙħ صر +ĠاÙĦÙħصر ÙĬØ© +ĠÐŁÐ¾ ÑįÑĤомÑĥ +ÙĨ صÙģ +ĠÙĪاÙĦ ÙĨ +Ġh Ãłi +à¸Ħ ิ +ĠApr ès +ì³ IJ +à¹Ģà¸ĭ ีย +×ĵ ×ŀ×Ķ +activ ité +à¸Ħิà¸Ķ วà¹Īา +ÑĤ ÑĢен +à¹Ģ ฮ +ãĥı ãĤ¤ +ãģĮ å¢ĹãģĪ +ен наÑı +Ġìĺ¤ ëĬĺ +ãĥ¢ ãĥ³ +Ġкон еÑĩно +ĠÙħÙĤ ابÙĦ +cl é +Ġh ü +Ġth ẳng +ìłģ ìĿ´ +ĠÐIJ лекÑģ +ĠÐIJлекÑģ ан +ĠÐIJлекÑģан дÑĢ +ãĥŀãĥ³ ãĤ·ãĥ§ãĥ³ +ãģ²ãģ¨ ãģ¤ +ãģª ãģĬ +à¹Ģà¸Īà¹īา à¸Ĥà¸Ńà¸ĩ +ëĵľ 리 +Ø´ اء +ĠsaÄŁ lık +ĠÅŁ imdi +×Ļ×IJ ׾ +تأ Ø«ÙĬر +Ø£ سب +أسب اب +ĠвÑĭполн ен +л ок +ש ×Ļ×ij×Ķ +Ġl ắm +ĠTr Æ°á»Ľc +Ġ×Ķ×¢ ׾ +리 를 +ĠÑĢ еж +ĠÑĢеж им +int é +inté gr +×Ĵ ׳×Ļ +ĠاÙĦØ´ عر +Ġmil hões +Ġpeque ño +ãĤ³ ãĥ¼ãĤ¹ +×ķ׼ ×Ĺ +à¹Ģà¸Ĭ à¹īา +شر ÙĤ +Ġh Æ°Æ¡ng +รัà¸IJ à¸ļาล +à¸ģล าย +à¸ģลาย à¹Ģà¸Ľà¹ĩà¸Ļ +Ġпод Ñħод +תש ×ķ×ij×Ķ +ãģıãģª ãģ£ãģ¦ +ĠاÙĦØ£Ùħ Ùħ +ĠH á»įc +ĠwspóÅĤ pr +ĠwspóÅĤpr ac +Ñĩ Ñĥв +ÑĩÑĥв ÑģÑĤв +ÃŃst ico +à¹Ģà¸ģ าะ +ìĽ Ģ +Ġназ ад +ãĤĭ ãĤĪãģĨãģ« +ĠС Ш +ĠСШ ÐIJ +м он +ĠAs ÃŃ +×ķר ×Ĵ +полн ен +×ŀס ׾ +×ŀ×¡×ľ ×ķ׾ +à¹Ģลืà¸Ń à¸Ķ +à¹Ģริà¹Īม à¸ķà¹īà¸Ļ +ĠاÙĦØ¥ Ùħ +ĠاÙĦØ¥Ùħ ارات +צ×Ķ ר +ãĥ¡ãĥª ãĥĥãĥĪ +ĠпоÑĤ ом +в из +ĠÙģ ترة +å¾Į ãģ® +ÐĿ ÐIJ +×ŀס ר +ÙĬر ÙĬ +pr é +Ġte ÅŁek +ĠteÅŁek kür +Ġöd eme +د اÙĨ +ãģ¾ ãģĹãģ¦ +缮 ãģ« +ĠÑĤ еÑĩение +l ard +lard ır +à¹Ģรา à¸Īะ +ס פ×Ļ +ĠÙĪÙĥ Ø°ÙĦÙĥ +Ġh át +Ġt á»Ļc +à¸Ħุ ย +Ġb ức +ØŃ ÙĬÙĨ +èģŀ ãģĦãģ¦ +Ùħؤ شر +ĠNh Æ° +Ġмен ее +ละ à¸Ħร +Ñģ ин +ĠÑĢ ек +ĠÑĢек л +ĠÑĢекл ам +ĠÙģ ÙĩÙĪ +Ġ׾ ×ĸ +×Ļ׳ ×ķת +ĠÅŁ art +ÑģÑĤав ка +Ġíı¬ íķ¨ +ãģ«è¡Į ãģı +ï¼ Ŀ +ĠпозволÑı еÑĤ +Ġת×ķ׼ ׾×ķ +ов ал +صÙĦ Ø© +Ġ׾ש ׳×ķת +ĠÐĺ гÑĢ +ÙħÙĨتج ات +Ġsat Ä±ÅŁ +Ñģ ко +ĠاÙĦØ«ÙĦاث اء +Ġ×Ķ×ĵ×ijר ×Ļ×Ŀ +ãģĹãģ¾ ãģĹãĤĩãģĨ +بÙĤ Ùī +åĬĽ ãĤĴ +ĠÃĩ ok +ãĥģ ãĥ¥ +à¹Ģà¸Ĭ ืà¹īà¸Ń +ยุ à¸Ħ +ศา ล +Ġק×ķ×ĵ ×Ŀ +×ĸר ×Ļ×Ŀ +ãģ® åł´åIJĪ +ĠìķĬ ìķĺ +ãģĤãĤĬãģ¾ãģĻ ãģĮ +×IJ שר +è¡Į ãģı +ãģ» ãģĭ +æ°Ĺ ãģ«ãģªãĤĭ +й деÑĤ +íķĺìĺĢ ëĭ¤ +ستÙħر ار +ĠÐŁÑĢ е +ĠÑģ боÑĢ +ĠìķĦ 무 +ç§ģ ãĤĤ +ع ص +Ġн иÑĩ +ĠниÑĩ его +ĠпÑĢи ем +ק ×ķ×ŀ +ĠìĪĺ ëıĦ +Ġì ¡´ +Ġì¡´ ìŀ¬ +ĠØ£ Ø«ÙĨ +ĠأثÙĨ اء +ĠÙĪاÙĦ ØŃ +ãģĮ ãģ§ãģįãĤĭ +Ġת ×Ķ +Ġת×Ķ ×Ļ×Ķ +ר ף +ĠÑģвÑıз и +×Ĵ שת +Ñģп екÑĤ +ס ×ij×Ļ×ij +ס×ij×Ļ×ij ×Ķ +ĠíķĦìļĶ íķľ +ت خصص +Ġж ив +Ġжив оÑĤ +ĠMay ıs +تع ا +تعا ÙĪÙĨ +ĠعÙĨ Ùĩا +ów ki +ĠاÙĦÙģÙĦسطÙĬÙĨ ÙĬ +ãģłãģijãģ§ ãģªãģı +ìĿ¸ ì§Ģ +ĠاÙĦس ÙĪد +ĠاÙĦسÙĪد اÙĨ +إجراء ات +Ġkö tü +Ġ×Ļ תר +×Ĵ ×Ļש×Ķ +Ġצ ×ķר×ļ +รà¸ĸ ย +รà¸ĸย à¸Ļà¸ķà¹Į +Ñħ оÑĤ +Ðł ÐIJ +ÙĪ Ø·ÙĨ +Ġsay ısı +ס ×Ĺר +Ùħ ÙĪÙĦ +ãĤĴæĮģ ãģ£ãģ¦ +ع اÙĨ +Ġt á»Ļi +ĠвÑĭ ÑĪе +Ġt ầm +ãĥĪ ãĥ¬ +×Ļצ ×ķ +ม ุม +س ÙĪد +ìłĦ ìŀIJ +ãĤµ ãĥŃãĥ³ +ìĤ° ìĹħ +ĠоÑģнов ан +Ø® Ùģض +רצ ×Ķ +بÙĬ ض +×ķÖ ¹ +ס×Ļ ×Ļ×¢ +Ġש ×IJ×Ļ +ĠاÙĦÙĤر Ø¢ÙĨ +ĠТак же +×ŀש ×ŀ×¢×ķת +س ÙĩÙĦ +Ġ×Ķ ׳×Ķ +ãĤĴ ãģĹãģ¦ãģĦãĤĭ +×Ļ ×Ļס +×Ķ ×ķ×IJ +ĠB ÃŃ +Ġмал о +ĠëĶ°ëĿ¼ ìĦľ +Ġר ×Ĺ×ij +ãģĮ é«ĺãģĦ +ÙĪ اس +ìĤ ¼ +׳ ×¢ +ãģ£ ãģ¡ãĤĥ +ĠT üm +à¸Ńีà¸ģ à¸Ķà¹īวย +ãģĹãģ¦ ãģıãģłãģķãģĦ +ÙĨØ´ اط +ãĥĹ ãĥ©ãĥ³ +али ÑģÑĮ +×ĵ ×ľ×ª +Ġwc zeÅĽ +ĠwczeÅĽ niej +ĠÑįÑĤ им +Ġthá»ĭ t +à¸ļ ัà¸į +à¸ļัà¸į à¸Ĭี +ãģļ ãģ£ãģ¨ +ÑĢ ин +Ġswo jÄħ +íķĺëĬĶ ëį° +Ġë§Įëĵ¤ ìĸ´ +تش Ùĥ +تشÙĥ ÙĬÙĦ +ائ Ùĩ +Ġ׾פ ×Ĺ×ķת +ãĥĭ ãĥ¥ +ãĥĭãĥ¥ ãĥ¼ãĤ¹ +׼×IJ ף +ãģ§ãģį ãģŁ +зв он +Ġsta ÅĤ +×Ĺ×ijר ת×Ļ +ĠØ£ عÙĦÙĨ +à¹ģà¸ļà¸ļ à¸Ļีà¹ī +بد Ø¡ +ãĤģ ãģŁ +Ġ×ŀש ×ŀ×¢×ķת +Ġ×ŀש×ŀ×¢×ķת ×Ļ +ör ü +Ġh ạnh +z ähl +ĠL ý +Ġ×ij ×Ķת +Ġ×ij×Ķת ×IJ×Ŀ +б аÑĢ +ì¦ Ī +ä»ĬåĽŀ ãģ® +Ġy ü +Ġyü ks +Ġyüks el +ãĤ½ ãĥ¼ +ãģĤ ãĤĮ +ת ׾×ŀ×Ļ×ĵ +ãģ¤ ãģª +×ij ׳×Ļ×Ŀ +Ġx ếp +ĠмÑĥж Ñĩин +ĠاÙĦÙĥ تاب +׼ ×ŀ×ķת +Ġç e +Ġçe ÅŁ +ĠçeÅŁ it +ĠçeÅŁit li +×ĵ ×Ļר×ķת +à¸ļุ à¸į +ĠاÙĦØ¥ ÙĦÙĥ +ĠاÙĦØ¥ÙĦÙĥ ترÙĪ +ĠاÙĦØ¥ÙĦÙĥترÙĪ ÙĨÙĬ +ĠباÙĦØ¥ ض +ĠباÙĦإض اÙģØ© +Ġyö nel +Ġyönel ik +mys ÅĤ +à¸Ķà¹īวย à¸ģาร +à¸ģาร à¸Ĺำ +ов Ñĭм +Ø£ زÙħØ© +æİ¢ ãģĹ +íļ ¨ +Ġ×ķ×IJ ×Ŀ +Ġnghi êm +ÑĪ ин +ка л +Ġcrian ças +èĩªåĪĨ ãģ§ +Ġн ай +Ġнай ÑĤи +ĠS á»ij +ĠÃ¶ÄŁrenc iler +ãĥ¶ æľĪ +Ñģ ан +ĠJ á +ĠkonuÅŁ ma +شر Ø· +ëĪ Ī +ar rière +ضر ÙĪرة +ãĥĶ ãĥ³ +×¢ שר +аÑĢ ÑĮ +جÙħ اع +Ġdé co +Ġ×Ļ×Ķ ×ķ×ĵ×Ļ +à¸ŀ ลาà¸Ķ +ĠÙĬ ÙĥÙĨ +Ġج اÙħعة +Ø· بÙĤ +Ġbo ÅŁ +×ķ ×ķ×IJ +×ŀ×ĵ ×¢ +ק×ij×ķצ ת +פ ×Ļר +jÄħc ym +ÙħØ´ ا +Ùħشا ÙĥÙĦ +צ פ×ķף +Ø¥ ست +×ŀ׼ ר +سÙħ ع +Ġкак ой +ÑĤ воÑĢ +ØŃ Ø¬ +Ùģر ض +пÑĢав лен +Ġник ак +Ġmi á»ĩ +Ġmiá»ĩ ng +ü ÃŁ +иÑĢов ал +׾ ×ŀ×ķת +次 ãģ® +ÙĦ Ø· +à¸ķ ัà¸Ļ +×Ķ ת×Ĺ×Ļ׾ +Ġfoto ÄŁ +ĠfotoÄŁ raf +طر ØŃ +à¸Ńà¸Ńà¸ģ à¹Ħà¸Ľ +Ġy ên +Ġп ок +Ġпок Ñĥп +ĠпокÑĥп а +ÑĨ Ñĥ +Ġкомп ÑĮÑİ +ĠкомпÑĮÑİ ÑĤеÑĢ +ĠاÙĦÙĥ رÙĬÙħ +تص Ùħ +تصÙħ ÙĬÙħ +Ġоказ а +Ġzar ówn +Ġzarówn o +ëĮĢ ì¶ľ +ãĤ»ãĥ³ ãĤ¿ãĥ¼ +Ġjako ÅĽci +æĤ © +æĤ© ãģ¿ +Ø£ÙĨ ÙĪ +Ø£ÙĨÙĪ اع +ë¹ ł +Ġìłķ ë§IJ +Ġk ẻ +ĠÑģай ÑĤа +Ġ×Ķ ער×ij +Ùĩ ز +pres ión +ĠÑģÑĤ ен +ãģ£ãģ¦ ãĤĭ +Ġhız lı +Ðļ ÐIJ +×ŀשפ ×Ĺת +ĠÙĨ Ùĩا +ĠÙĨÙĩا ÙĬØ© +ãģ¾ ãģĦ +о ÑħÑĢан +ร à¹īà¸Ńย +ล ึà¸ģ +ĠÙĪب اÙĦ +ãĤĤãģ® ãģĮ +ר׼ ×Ļ×ij +ãĤ¤ ãĥ¤ +س ؤ +سؤ اÙĦ +ĠÙĦØ£ÙĨ Ùĩ +ĠkonuÅŁ tu +Ðļ ÑĥпиÑĤÑĮ +Ġש×IJת ×Ķ +ĠÙĪاÙĦ س +Ġmożliwo ÅĽci +Ġpró b +ëĶ ° +ãģ© ãĤĮ +ĠÐľ ин +ĠоÑĢганиз м +ãģ«å¯¾ ãģĻãĤĭ +ĠPr é +Ġpriv é +ch è +ãģĦãģŁãģł ãģį +สà¸Ļุ à¸ģ +ajÄħ ce +ĠD zi +ĠDzi ÄĻki +ÅĤat w +r än +rän k +æĿ¥ ãģŁ +Ġ×Ķ×Ļ×Ķ ×ķ×ĵ×Ļ +ãĤ¬ ãĥ¼ +ĠÑĢаР´ +ĠÑĢад и +к ÑĤив +Ø£ Ùĩد +Ø£Ùĩد اÙģ +ש ×IJ×Ļר +ãģ¦ ãģĦãģªãģĦ +Ġfr üh +Ġок ол +Ġокол о +Ġreg ião +ĠÑĩиÑģ ле +Ġpon iew +Ġponiew aż +ìĦ¼ íĦ° +Ġb ầu +Ġê · +Ġê· ľ +Ġê·ľ ìłķ +ĠH òa +ĠÑĤ оÑĤ +ãĤĤ å¤ļãģĦ +ĠاÙĦإسÙĦاÙħ ÙĬØ© +ãģĭ ãģĦ +Ñį н +ĠÑĥказ ан +ĠÑĤак ое +ï¼ ³ +ëĮĢ íķĻ +Ġgen iÅŁ +ĠاÙĦØ® ÙĬ +ĠاÙĦØ®ÙĬ ارات +ãĤĴè¡Į ãģĨ +ש ×ŀ×Ķ +ĠLÃł m +ÙĪÙĨ ÙĬ +Ġ×IJ ׾×Ļ×ķ +Ä ĺ +à¹Ħมà¹Ī สามารà¸ĸ +人 ãģ¨ +بر ز +×Ļס ×ķ×ĵ +×Ĵ ׾×Ļ +ĠÙĬ ÙĨا +ĠÙĬÙĨا ÙĬر +ĠкаÑĢÑĤ ин +Ġt ôn +à¹Ģ à¸ģร +à¸Ħ à¸Ķี +Ġ׾×IJ ×ķר×ļ +ãĤĤãĤī ãģĨ +ãģĭ ãģĭãĤĭ +ани и +Ġara ÅŁtırma +ÙĦاØŃ Ø¸ +ãģĦ ãĤĦ +ĠT Ãłi +Ġ à¸Ļà¸Ńà¸ģà¸Īาà¸ģ +Ġà¸Ļà¸Ńà¸ģà¸Īาà¸ģ à¸Ļีà¹ī +ĠÄIJ ảng +ãģ£ãģ¦ ãģįãģŁ +Ġà¸ĭึà¹Īà¸ĩ à¹Ģà¸Ľà¹ĩà¸Ļ +Ġt ả +Ġmożliwo ÅĽÄĩ +ĠS ản +ĠÄ° ki +Ġc ắt +س Ø£ÙĦ +Ġbak ım +Ø´ ب +à¸ķ ีà¹ī +à¸ŀ ยาย +à¸ŀยาย าม +สั à¸Ľ +à¸ªà¸±à¸Ľ à¸Ķา +à¸ªà¸±à¸Ľà¸Ķา หà¹Į +ë° Ģ +еÑĢ Ñĭ +Ġc ánh +Ġthu ế +ت بع +ãģ«åħ¥ ãĤĮ +Ñİ ÑģÑĮ +íļĮ ìĿĺ +ç°¡ åį +ç°¡åį ĺ +ç°¡åįĺ ãģ« +Ġtr úc +ĠاÙĦÙĥ ÙĪÙĬ +ĠاÙĦÙĥÙĪÙĬ ت +ãĤıãģij ãģ§ãģĻ +ĠÑģв об +ĠÑģвоб од +ĠÑĥÑĩаÑģÑĤ ник +สิ à¹īà¸Ļ +ĠпÑĢо ÑĦеÑģÑģиона +ĠпÑĢоÑĦеÑģÑģиона лÑĮн +Ñģп оÑĢ +×Ĺ ×ķ×ij×Ķ +Ùħع ÙĨÙī +ĠاÙĦÙģ ترة +สูà¸ĩ สุà¸Ķ +ãĤı ãģļ +ĠÄij è +ĠÄijè n +æ¯Ķ ãģ¹ +า à¸ĺิ +Ġmoż emy +à¹ģ à¸ĭ +à¸Īะ à¹Ħมà¹Ī +Ġs ắp +Ðļ Ðŀ +Ġprá ctica +ÙĪÙĥ اÙĦØ© +è¾¼ ãĤĵãģ§ +ológ ica +Ġе Ñī +ĠеÑī Ñij +تع دÙĬÙĦ +ĠØ£ Ùĥد +Ġצר ×Ļ׼ +Ġצר×Ļ׼ ×Ļ×Ŀ +Ø« Ùħ +Ġк ÑĢÑĥ +ĠкÑĢÑĥ п +×ij×Ļק ×ķרת +Ġì¡° ê¸Ī +ãģ¨ãģį ãģ¯ +Ġb ạc +ĠÑĢаÑģ пол +ĠÑĢаÑģпол ож +ĠÑĢаÑģполож ен +ز ÙĬÙĨ +ĠÐļ ÑĢоме +ĠاÙĦÙĨ ظر +×Ķ ×ķ×ĵ +ĠاÙĦس بت +ãģ¨æĢĿ ãģĦ +Ġpa ÅĦst +ĠpaÅĦst w +ĠÙĦÙĬ ست +ĠбÑĥд Ñĥ +à¸Ĺัà¸Ļ à¸Ĺี +ร าม +ØŃ ØµÙĪÙĦ +ãģĹãģ¦ãģıãĤĮ ãĤĭ +ĠاÙĦØ¥ سرائÙĬÙĦ +ĠاÙĦإسرائÙĬÙĦ ÙĬ +ãģĵãĤĮ ãģ¾ãģ§ +ìĤ¬ 를 +Ġs ürü +à¹Ģว à¸Ńรà¹Į +à¹Ģà¸ĭ à¸Ńรà¹Į +Ġutilis é +ĠÑģиÑģÑĤем а +Ġdw ó +Ġdwó ch +Ġpróp rio +Ġëĵ± ìĿĦ +arr êt +ĠЧ а +×IJ×ŀ ׳×ķת +عار ض +à¹Ģà¸ģม สà¹Į +Ġ׾×Ķ ×ij×Ļף +Ġ׾ ×ij×Ĺ +Ġ׾×ij×Ĺ ×ķר +สา à¸Ĥา +ĠÐľÐ¾Ñģк ве +ب عد +ĠاÙĦÙĤر ار +ĠÄIJ á»ĭa +Ġ×Ĺ ×Ĵ +Ùģ تر +ÙĪÙĨ Ø© +Ġ×Ķ×ĸ ×IJת +å¸Ĥ ãģ® +ãģ» ãģĹãģĦ +Ġ×ij×¢ ×Ļר +ĠÑĤеп еÑĢÑĮ +ìĬµ ëĭĪê¹Į +à¹Ħม à¹Īว +à¹Ħมà¹Īว à¹Īา +à¹Ħมà¹Īวà¹Īา à¸Īะ +×ŀ ×IJ×Ķ +æĥħ åł± +æĥħåł± ãĤĴ +غ ÙĨ +Ġпо Ñı +ĠпоÑı ви +éģİ ãģĶ +تش غ +تشغ ÙĬÙĦ +в ел +Ġ×Ĺ ×ŀ +ãģ¨ãģªãĤĬ ãģ¾ãģĻ +Ġra ÄŁ +ĠraÄŁ men +ãģĭ ãģ©ãģĨ +ãģĭãģ©ãģĨ ãģĭ +ен ко +ì§Ģ ê³ł +Ġ×IJ׾ ×Ļ×Ķ +ĠØ£ ÙĦ +à¸Īำ หà¸Ļ +à¸Īำหà¸Ļ à¹Īาย +nız ı +Ġ׾ק ×Ĺת +Ø£ ÙĩÙħ +Ø£ÙĩÙħ ÙĬØ© +ت غÙĬر +ש ×Ĺר +ס×ķפ ר +×ĵ ×Ļר +èī¯ ãģĭãģ£ãģŁ +×ŀ׾×Ĺ ×ŀ×Ķ +ÑģÑĤв ие +ÑĤ ÑĢаÑĤ +ĠاÙĦØ£ Ø® +ĠاÙĦأخ ÙĬرة +ĠاÙĦØŃ ØµÙĪÙĦ +Ġcréd ito +צ ×Ļ×¢ +ãĥ¬ ãĥĻãĥ« +بر ÙĬ +ëIJ IJ +ãģł ãģ£ãģ¦ +Ġreal tÃł +س Ùģر +×ķ׳ ×ķ +×Ĵ ×ķ×ĵ +×Ĵ×ķ×ĵ ׾ +ฮ า +ãģĹãģ¦ ãģĬãĤĬãģ¾ãģĻ +Ġg Ãł +Ġ׾×ij צע +å¼ķ è¶ĬãģĹ +Ġ×ŀ ×Ļ׾×Ļ +Ġ×ŀ×Ļ׾×Ļ ×ķף +Ùħ در +Ùħدر سة +פ ×ķ×ĺ +à¸Ļà¹īำ มัà¸Ļ +ëģ Ŀ +ع Ùĥس +ĠÙĤ ض +ĠÑĢÑĭ б +خط Ø· +×ŀ×ķס ×ĵ +Ġ׼׾ ׾×Ļ +ĠкоÑĤоÑĢ ое +צ×Ļ ×ķף +ĠмеÑģÑĤ а +ãģĭ ãģ¤ +г ÑĢÑĥпп +׾ ×Ļ׾ +ת ×ķ×IJר +ë³µ ì§Ģ +à¹ģà¸ľ à¹Īà¸Ļ +Ġ×ij×¢ ת +æĻĤéĸĵ ãĤĴ +ï¼ £ +ãģ¨ãģĦãģĨãģĵãģ¨ ãģ§ +Ġ׾×Ķ ק +Ġ׾ ×ĸ×Ķ +ĠìłĢ ëĬĶ +ĠاÙĦØ¥ رÙĩاب +ĠìŀĪëĬĶ ëį° +ĠÑĤ огда +Ġ×Ķ צ×Ļ +×ķ׾ ×ĺ +Ġר פ×ķ×IJ×Ļ +ãģĵãģ¨ ãģ§ãģĻ +ĠÄij ÃŃch +ØŃ ÙĬا +Ġ×Ķ×ŀש ×Ĺק +ãģľ ãģ² +Ġ×ŀ×IJ פשר +ãģ¿ ãģ¾ãģĹãģŁ +ĠاÙĦØ£ÙħÙĬر ÙĥÙĬ +Ùħج تÙħع +Ġس اب +Ġساب ÙĤ +׼ ×Ļ׾ +Ạ¾ +ãĥª ãĤ¹ãĥĪ +Ġì ĥ +Ġìĥ Ī +ĠìĥĪ ë¡ľ +ĠìĥĪë¡ľ ìļ´ +ĠD á»ĭch +à¹Ģหมาะ สม +ĠاÙĦÙĨ بÙĬ +׾ ׾ +ÙĨ ع +Ðĵ лав +Ðĵлав наÑı +Ùħر ض +Ġ×ķ ×ĵ +ت ÙĤÙĬ +تÙĤÙĬ ÙĬÙħ +Ġb ảng +ĠÙģ ÙĤاÙĦ +×¢ ×ŀ×Ļ +д ÑĢа +Ġsu á»ijt +سر عة +Ġc á»Ń +Ġ×Ķ ×Ļ×Ĺ×Ļ×ĵ +سع ÙĬد +à¸Ńา à¸Ĭีà¸ŀ +Ġس ÙĪاء +ãĤ½ ãĥķãĥĪ +Ġл иÑĩно +ĠÐļ оÑĢ +اÙĩ تÙħ +اÙĩتÙħ اÙħ +à¸Ń à¸Ķี +à¸Ńà¸Ķี à¸ķ +ãģIJ ãĤīãģĦ +Ġiht iya +Ġihtiya ç +ãģ¾ãģ§ ãģ® +ìĭľ ìĬ¤ +ìĭľìĬ¤ íħľ +ÑĢÑĥ ÑĪ +ãĤĦ ãģ£ãģ± +ãĤĦãģ£ãģ± ãĤĬ +к еÑĢ +Ġ ży +Ġży w +кл он +Ġl ượt +à ¾ +да Ñĩи +tür k +غ ÙĪ +ĠигÑĢ ок +Ġph ê +Ġש ×¢×ľ +ĠاÙĦÙħ دÙĨÙĬ +ĠìŬ룬 ë¶Ħ +ער ×Ļ×Ŀ +Ñħод ÑıÑĤ +Ġx ứ +ÐĹ Ð° +ĠÙģ رص +à¸Īะ à¸Ĺำà¹ĥหà¹ī +íģ ´ +×¢ ×ij×ķר +à¹Ģหลà¹Īา à¸Ļีà¹ī +èĢĥãģĪ ãĤĭ +ÑĢ еÑģÑĤ +н нÑĭй +Ġc ầm +دا Ø®ÙĦ +ĠÙħÙĦÙĬ ار +ĠÐIJ л +ĠвÑĢем ен +à¸Ĭà¹Īวย à¹ĥหà¹ī +ר×Ļ ×ķת +ëĵ ¯ +飲 ãģ¿ +׳ ׾ +שת ×£ +ĠاÙĦسعÙĪد ÙĬ +u ÃŁ +ìĿ¸ ëį° +ĠìĿ¼ ë°ĺ +ÅĤ ÄĻ +Ġm á»iji +×ŀ ×Ļ׳ +ĠاÙĦØ£ Ø·ÙģاÙĦ +Ġçı kan +é cole +ק ×Ļש +ק×Ļש ×ķר +ĠоÑģ ÑĥÑīеÑģÑĤв +ĠоÑģÑĥÑīеÑģÑĤв лÑı +×ij ×IJר +à¹Ħà¸Ľ à¸Ķà¹īวย +Ġ×¢ ×ķ׾×Ķ +à¸ģà¹ĩ à¹Ħมà¹Ī +ãĥ¢ ãĥĩ +ãĥ¢ãĥĩ ãĥ« +تØŃ ÙĪÙĦ +Ġод ного +ת×Ĺ×Ļ׾ ת +Ġت Ø® +Ġch cia +Ġchcia ÅĤ +ãĥIJ ãĥ³ +èĢħ ãģ¯ +ĠÙħ ØŃÙĦ +Ñģл ож +Ñģлож н +Ġt ÄĻ +Ġçı kt +Ġçıkt ı +ĠC Æ¡ +à¹Ħà¸Ķà¹ī à¹Ģลย +ır ken +à¹Ģà¸Ĥà¹īา สูà¹Ī +ÙħØŃ Ùĥ +ÙħØŃÙĥ ÙħØ© +à¸Ħุ à¹īม +à¸Ļà¹Īา à¸Īะ +лÑİ Ð´ +де ÑģÑı +деÑģÑı ÑĤ +ĠлÑİб ой +تØŃر ÙĬر +צע ×ĵ +Ġе Ñij +ĠاÙĦØŃ ÙĥÙħ +Ġص باØŃ +à¹Ģà¸ļ à¸Ńรà¹Į +Ġróż nych +ги б +ĠÑģ оÑĤ +ĠÑģоÑĤ ÑĢÑĥд +ĠÑģоÑĤÑĢÑĥд ник +ĠобÑĬ ем +פ ×ĺר +ãģĻãģĶ ãģı +ãģ«éĸ¢ ãģĹãģ¦ +в ол +Ø« ÙħاÙĨ +Ġd ần +æĬ ľ +æĬľ ãģij +Ġ×¢ ש +Ġעש ×ķ×Ļ +ס ×ķף +ãģªãģ® ãģ§ãģĻ +ãģ¯ ãģ©ãģĨ +×ŀ×¢ ר×ij +ï¼ ° +Ùħ صر +ÙħÙĨ اسب +ÙħÙĨاسب Ø© +ä¸Ĭ ãģ® +×IJ×Ļש ×ķר +ĠìĦ¤ ì¹ĺ +×ŀ×ĵ×Ļ׳ ×ķת +×ŀר ת +ãĤĭ ãģ®ãģĮ +د Ùİ +ĠاÙĦشر Ùĥات +ìĭľ ê°Ħ +ĠÑĢеÑĪ ение +ãģĻãĤĭ ãģ®ãģ¯ +ĠìŀIJìĭł ìĿĺ +׾ ×ŀ×ķ +ãģ¨ãģĵãĤį ãģ§ +Ġק צר +Ġmã i +Ġkü ltür +ãĥ©ãĤ¤ ãĥĸ +à¸ľà¸¹à¹ī หà¸įิà¸ĩ +æĻĤéĸĵ ãģĮ +клÑİÑĩ и +diÄŁ iniz +มาà¸ģ à¹Ĩ +تØŃ ÙħÙĦ +Ġh ạt +ãĤ¦ ãĤ£ +п ле +×ŀ ׾×IJ +ÅĤ ó +Ġg á»ijc +Ġ×IJ ×ķ×ĵ×ķת +หว าà¸Ļ +ĠاÙĦ ÙĪز +ĠاÙĦÙĪز راء +ëĵ¤ ê³¼ +Ġص ØŃ +ĠصØŃ ÙĬÙģØ© +Ġм м +تد Ø®ÙĦ +Ġpersön lich +Ġز ÙĬ +ĠزÙĬ ادة +ãĤ· ãĤ¢ +Ġng ắn +à¸Ħล ิà¸ģ +Ġs ông +Ġtü ket +Ñį ÑĦÑĦ +ÑįÑĦÑĦ екÑĤ +ש ×Ļ×ij +Ġا عت +ت ض +تض ÙħÙĨ +ĠاÙĦÙħØ´ رÙĪع +Ġprodu ção +ĠпÑĢимен Ñı +ни ÑĨÑĭ +주 ëĬĶ +ر Ùı +Ġm Æ¡ +Ġhayat ı +ëŁ ½ +Ġü cret +Ġyan ında +Ġpr ática +×ij×Ļק ×ķר +Ãľ N +Ñģ оÑĤ +ãĤıãģij ãģ§ +Ġдол го +ת ׼×ķ +ĠìķĦ ëĭĮ +ë į°ìĿ´ +Ġç iz +Ġcho Äĩ +Ġ×Ķ ×Ļת +Ġ×Ķ×Ļת ר +Ġso át +׼ ×ij×ĵ +à¹Ģล à¹Īา +Ġд еÑĢ +ĠдеÑĢ ев +ãĤĴ åħ¥ãĤĮ +×Ĺ ×ķס +×Ĺ×ķס ר +ج ÙĬÙĨ +t ón +onn é +Ġпол ноÑģÑĤÑĮÑİ +人 ãģŁãģ¡ +Ġpr êt +ëł ¸ +Ġdéc embre +cı lar +Ġת ת +Ġê²½ìļ° ìĹIJëĬĶ +ÙĪ عد +è¦ĭ ãĤĭ +วิ à¸Īัย +ë ¶Ī +ز ÙĪا +زÙĪا ج +d ì +ãģ§ãģĻ ãĤĪ +Ġвод о +ĠÙĬ ÙĪجد +Ñģ оÑģÑĤоÑı +Ðŀ С +ĠÄIJ ó +׊פש +Ġצ ×Ļ×ij×ķר +ĠاÙĦÙĤ Ø· +ĠاÙĦÙĤØ· اع +Ġиме ÑİÑĤ +Ġph áºŃn +×Ľ×¡ פ×Ļ +полн иÑĤелÑĮ +éĻIJ ãĤĬ +ĠÑģ ÑĢав +ĠÑģÑĢав н +ÙħاÙĦ Ùĥ +×ĵר ×ķ×Ŀ +çļĨ ãģķãĤĵ +ØŃÙĤ ÙĤ +à¹ģหล à¹Īà¸ĩ +ĠاÙĦر سÙħÙĬ +оÑĩ ки +×ĺ ×ij×Ĺ +Ġcan lı +Ġ׾ ׾ +Ġ׾׾ ×ŀ×ķ×ĵ +×ŀ×ij ×ķ +ת ׼ +×ª×Ľ ׳×Ļת +ĠاÙĦÙħ شار +ĠاÙĦÙħشار ÙĥØ© +Ä° Åŀ +ĠسÙĬ اسÙĬ +в олÑĮ +ĠÑģ пÑĢав +æĿ¥ ãģ¦ +פ×ķר ×ķ×Ŀ +สำ à¹Ģรà¹ĩ +สำà¹Ģรà¹ĩ à¸Ī +ĠÅŁ öyle +Ġzosta ÅĤa +ĠH ü +ר ×ķש +د ÙĦÙĬÙĦ +ÑĢи д +ש ף +×ŀק ×ķר +ĠÑĥ Ñĩ +ĠÑĥÑĩ еб +ĠÑį ÑĤа +ков а +à¸ķà¸Ļ à¹Ģà¸Ńà¸ĩ +ÙĨ ÙIJ +à¸Ńีà¸ģ à¸Ħรัà¹īà¸ĩ +ระ à¸ļุ +Ġd ữ +ĠاÙĦØŃ Ø§ÙĦÙĬ +׼ ×ķ׼ +׼×ķ׼ ×ij +Ġ×ŀ×IJ שר +Ġtr ụ +ÑĤел ем +Ġв ли +Ġвли Ñı +Ġש×IJת ×Ŀ +Ġuw ag +Ġuwag ÄĻ +×ĺ ×Ļת +×IJ ×ĵ×Ŀ +à¸Ķ ุ +Ġ×Ķ×IJ ׾×Ķ +Ġkar Ä±ÅŁ +ĠÄIJ á»iji +да ÑİÑĤ +ãģªãģ® ãģ« +Äħ cych +à¹Ģà¸Ļ à¹īà¸Ļ +ãģĹãģ¦ ãģĹãģ¾ãģĨ +int érieur +ĠfÃŃs ica +ĠÐŁ ол +ãģĹãģ ķ +à¸Ĺำ à¹Ħม +ĠL âm +ĠاÙĦÙħ سÙĦÙħ +ĠاÙĦÙħسÙĦÙħ ÙĬÙĨ +ص ØŃØ© +ìĹ Ħ +à¹Ģà¸Ķà¹ĩ à¸Ķ +ĠÑĥ ÑĩеÑĤ +â Ìģ +Ġب ÙĦا +ĠاÙĦاجتÙħاع ÙĬ +פרס ×Ŀ +ãĥķ ãĥ© +ĠÐļ огда +mie ÅĽci +ĠبÙĬÙĨ Ùħا +Ġ×ŀ×IJ ×ŀר×Ļ×Ŀ +Ġ×ij×IJ ×ĸ×ķר +×ķש ×Ļ×Ŀ +ĠÑģдел а +entr ée +à¹Ģ à¸Ħà¹īา +Ñĥг л +ĠاÙĦÙģ ÙĨÙĬ +ĠÐĴ оÑĤ +à¸Ĺีà¹Ī มา +×ķצ ×Ĵ +ÙĤد رة +Ġëª © +Ġ목 ìłģ +íıī ê°Ģ +ĠاÙĦØ£ ربع +ĠاÙĦأربع اء +פס ×Ļק +ĠÑıвлÑı ÑİÑĤÑģÑı +ب ÙĪÙĨ +ì° ¾ +×ŀ×¢ ר׼ +×ŀ×¢×¨×Ľ ×ķת +ãĤ· ãĤ§ +ĠباÙĦ Ø£ +íĸĪ ëįĺ +ĠاÙĦبر ÙĨاÙħج +ĠاÙĦØ£ ØŃد +Ġm Å© +ĠmÅ© i +п аÑĤ +ب Ø« +ĠÑĨ енÑĭ +Ġ×ijת ׾ +è¨Ģ ãĤıãĤĮ +ĠاÙĦÙħ جاÙĦ +ĠìĦ¸ ìĥģ +Ġ×Ĵ ×ķפ +ĠнаÑĪ ей +Ġкомп аниÑı +б ин +öl ü +×Ļ ×Ļ×ĺ +Ġ×ŀס פ×Ļק +ยัà¸ĩ à¸Ħà¸ĩ +ĠЧ и +Ġан ÑĤи +ĠÑģÑĢед и +สà¹Īวà¸Ļ à¹ĥหà¸įà¹Ī +оÑĩ ка +íĬ¹ ë³Ħ +ว à¹Īาà¸ĩ +гоÑĢ од +با Ùĥ +à¹Ģส ีà¹Īย +à¹Ģสีà¹Īย à¸ĩ +ãĤĤãĤī ãģĦ +ק ×ķ×Ŀ +ãģĽ ãģļ +ĠاÙĦÙĤ اÙĩرة +Ġ×ij ׼×ļ +Ùħشار ÙĬع +باØŃ Ø« +Ġпо Ñĩ +ĠпоÑĩ ÑĤи +ĠÑĦоÑĢм а +S Ä° +Ġ×ŀצ ×Ļ×¢ +ล ื +ลื ม +ĠÑĤ еÑĢ +ĠÑĤеÑĢ ÑĢиÑĤоÑĢ +ĠÑĤеÑĢÑĢиÑĤоÑĢ ии +Ġв меÑģÑĤ +ĠвмеÑģÑĤ е +dıkl arı +op ération +à¹Ĥ ห +ص دÙĬ +صدÙĬ ÙĤ +íĸī ìłķ +تج ا +تجا ÙĪز +Ġsu ç +Ġar ty +Ġarty ku +Ġartyku ÅĤ +ãĤ·ãĥ§ ãĥĥãĥĹ +ש פ +שפ ×Ļ×¢ +Ġ×Ķש ×Ļר×ķת +à¹ģà¸ĸ ม +ë¸ Ķ +Ġuk ÅĤad +Ġ×ķ ׼×Ļ +หล าà¸ģ +หลาà¸ģ หลาย +æĸ¹ ãĤĤ +Ġpodr óż +ĠE ÄŁer +Ġком наÑĤ +ĠÑģам ÑĭÑħ +Ġв кÑĥÑģ +б еж +Ġ×ij ק×ķ +æİĽ ãģij +ãģ¿ ãĤĭãģ¨ +ĠiliÅŁ kin +ĠÙĬ عÙħÙĦ +Ġпод аÑĢ +Ġyaz ılı +ãĤĴ å¾Ĺ +Ġwyst ÄĻp +à¸Ĺีà¹Ī à¹ĥà¸Ĭà¹ī +ØŃاد Ø« +ÙĪ ÙĬد +кÑĥ лÑĮÑĤ +кÑĥлÑĮÑĤ ÑĥÑĢ +à¸ģาร à¹ģà¸Ĥà¹Īà¸ĩ +à¸ģารà¹ģà¸Ĥà¹Īà¸ĩ à¸Ĥ +à¸ģารà¹ģà¸Ĥà¹Īà¸ĩà¸Ĥ ัà¸Ļ +ÙħÙĪ ظ +ÙħÙĪظ Ùģ +ÙĬÙħ ÙĬ +ãĤĵãģ§ãģĻ ãģĮ +diÄŁ im +diÄŁim iz +ĠÐŁ еÑĢ +ĠÐŁÐµÑĢ в +Ġm ão +ĠÑģ ез +ĠÑģез он +Ġ×Ķ×ŀ ×¢ +Ùħ جÙħÙĪعة +ĠинÑĦоÑĢм аÑĨии +i ếc +ã ng +ĠÄij ấy +ãģĶ ç´ +ãģĶç´ ¹ +ãģĶç´¹ ä»ĭ +Ġad ım +à¹Ħ หล +Ġп ÑĢакÑĤи +ĠпÑĢакÑĤи Ñĩ +ĠпÑĢакÑĤиÑĩ еÑģ +ĠпÑĢакÑĤиÑĩеÑģ ки +ĠاÙĦÙĨ Ùģس +ĠÑĢабоÑĤ е +ÙĦÙĬ Ùģ +ĠاÙĦجÙĨ ÙĪب +Ġвод Ñĭ +ì¹ Ļ +Ġм иÑĢа +ĠÄij ừng +ĠпÑĢоÑĤив о +ĠÑģÑĤÑĢан Ñĭ +ล ู +ìĤ ¶ +kre ÅĽl +Ġbul und +Ġbulund uÄŁu +à¹ģ สà¸Ļ +ãĤ± ãĤ¢ +ת×Ĺ ×ķ×ŀ×Ļ +ר׼ ×Ķ +Ġ׾ק ×ķ×Ĺ +Ġ׾ק×ķ×Ĺ ×ķת +Ġ×Ľ×ª ×ķ×ijת +ĠÙĦ ÙĥÙħ +ب شر +Ġr Ãłng +Ġ×ŀ×Ķ ×ŀ +Ġ×IJ×Ĺר ×ķת +Ġб он +Ġбон ÑĥÑģ +ï½ Ĺ +à¹ģ ยà¸ģ +ãģĤãģªãģŁ ãģ® +ĠÑĥÑĩаÑģÑĤ ие +ĠE yl +ĠEyl ül +ĠçalÄ±ÅŁmalar ı +Ø® طر +ìĿ ½ +à¸ģาร à¹ĥà¸Ĭà¹īà¸ĩาà¸Ļ +Ġана лиз +תק ×ij׾ +ни ем +ĠÄ° ns +ĠÄ°ns an +ĠبÙĪ اس +ĠبÙĪاس طة +Ġ׳ ×Ľ×ł×¡ +Ġ×Ķ×ŀ ×Ļ×ĵ×¢ +Ġç o +Ġço ÄŁu +á» ĺ +ĠêµŃ 민 +ãĤĤ ãģĦãģĦ +Ġ׼ ׾×Ļ +ĠÑģÑĢед не +g ÅĤo +gÅĤo ÅĽ +Ġneg ó +Ġnegó cio +ĠÑĢ егиÑģÑĤ +ĠÑĢегиÑģÑĤ ÑĢа +ĠÑĢегиÑģÑĤÑĢа ÑĨии +Ġtr á»ĵng +ĠпÑĢ Ñı +ĠпÑĢÑı мо +ëłĪ ìĿ´ +Ġk ém +к ле +à¸Ļำ มา +ĠÑĦ ин +ĠÑĦин анÑģ +ĠÑĦинанÑģ ов +Ġki á»ĩm +ยัà¸ĩ à¹Ħ +ยัà¸ĩà¹Ħ à¸ĩ +ย ิà¸ĩ +à¹Ĥ à¸Ľ +ĠполÑĥÑĩ ил +×Ļ×ĸ ×Ŀ +à¹ģละ à¸Ħวาม +Ġво обÑīе +ص ÙĬر +ãĥı ãĥ³ +ĠاÙĦÙĤ اد +ĠاÙĦÙĤاد Ùħ +Ġب دÙĪÙĨ +ع ظÙħ +ת ׳×ķ×¢ +×ª×ł×ķ×¢ ×Ķ +Ø£ ÙħÙĦ +ãģķ ãģĪ +ÑĤ ем +ÑĤем пеÑĢ +ÑĤемпеÑĢ аÑĤÑĥÑĢ +Ġ׾ ×Ļצ×ķר +Ġr ÄĻk +ر سÙĦ +ìŀIJ 를 +Ġ×Ļצ ×Ļרת +ÙĨ بÙĬ +Ñĩ наÑı +تØŃ ÙĦÙĬÙĦ +Ġм ик +Ġмик ÑĢо +ĠS öz +Ġfor ça +Ñģ он +ĠاÙĦع را +ĠاÙĦعرا ÙĤÙĬ +ĠH á»ĵng +ãģĻãĤĭ ãģŁãĤģãģ« +à¸Ĺีà¹Ī à¸Ńยูà¹Ī +Ġ×ķ×IJ ×£ +ص ÙĬد +ĠìķĬ ê³ł +ร ัà¸ĩ +ĠاÙĦت ÙĪاصÙĦ +à¹Ģม à¸ķร +Ñĥ ÑģÑĤÑĢой +ÑĥÑģÑĤÑĢой ÑģÑĤв +m ıyor +Ġبا سÙħ +Ġ×ķ ׼×ķ +ĠG ül +á» IJ +Ãī tat +غ اÙĦ +Ø¥ ÙĨØ´ +Ø¥ÙĨØ´ اء +T Ä° +à¸Ĥà¹īา ม +Ġtro ch +Ġtroch ÄĻ +Ø¥ ص +إص ابة +ĠØ« اÙĨÙĬ +ĠاÙĦص ØŃØ© +Ġ×ĸ×Ķ ×ķ +jÄħ cej +ãĥĢ ãĥ³ +ìĿ¸ ìĿ´ +Ġв олоÑģ +ëIJĺ ë©´ +Ġzak ÅĤad +ãģĻ ãģĵãģ¨ +以ä¸Ĭ ãģ® +Ġ×Ķ×ŀק ×ķ×Ŀ +ÙħØ´ اÙĩ +ÙħشاÙĩ دة +Ñĩ ив +ب Ø´ +ย à¹īาย +Ġsür dür +ĠN ẵ +ĠNẵ ng +ĠигÑĢ аÑĤÑĮ +Ġê·¸ëŁ¬ ë©´ +ãĥķ ãĥ« +ล à¹Īะ +Ġtend rá +Ġb Ãły +à¹Ģà¸Ľà¹ĩà¸Ļ à¸ľà¸¹à¹ī +Ġok o +Ġoko ÅĤo +w ÅĤa +wÅĤa ÅĽci +wÅĤaÅĽci w +æĢĿ ãĤı +ĠYa ÅŁ +ĠB á»ĩnh +íı Ń +بÙĬ د +קר ף +à¹Ģศ ร +à¹Ģศร ษ +à¹Ģศรษ à¸IJ +à¹Ģศรษà¸IJ à¸ģิà¸Ī +ĠاÙĦØ£ ÙĪرÙĪ +ĠاÙĦØ£ÙĪرÙĪ بÙĬ +fl äche +ä¹Ĺ ãĤĬ +Ġb á»ģn +Ùĩ ب +æľĢ ãĤĤ +Ġsa ç +à¸Ńำ à¹Ģà¸ł +à¸Ńำà¹Ģà¸ł à¸Ń +ĠØ£ ج +ĠاÙĦد اخÙĦ +ĠاÙĦداخÙĦ ÙĬØ© +×ĺ ×ķ×ij +ãĤĤ ãģªãģı +Ġли ÑĨа +à¹ģลà¹īว à¸ģà¹ĩ +×ĸ׼ ×Ļר +Ġqu Ãł +ĠÙĥ Ø°ÙĦÙĥ +صØŃ Ùģ +ĠÃĤ u +ÙĪب ا +à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļ à¹ģà¸Ľà¸¥ +à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļà¹ģà¸Ľà¸¥ à¸ĩ +à¸ķัว à¸Ńยà¹Īาà¸ĩ +Ġráp ida +Ġtas ar +Ġtasar ım +ĠعÙĦÙĬ ÙĩÙħ +ס ×ķ׾ +c ılı +cılı k +Ġر غÙħ +ìĭľ íĤ¤ +Ġ×IJ׾ ק +Ġ×IJ׾ק ×ĺר +Ġ×IJ׾ק×ĺר ×ķ׳×Ļ +à¹ģà¸ļ à¹Īà¸ĩ +Ġh ạng +ãģ£ãģ¦ ãģıãĤĮ +ĠÙĨ تÙĬ +ĠÙĨتÙĬ جة +ıkl ı +غ اÙĨ +à¸Ĥà¹īà¸Ń à¸Ħวาม +à¸Ľà¸¥ าย +ĠØ£ Ùħس +à¸Ĺีà¹Ī à¹Ģà¸ģีà¹Īยว +à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยว à¸Ĥ +à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยวà¸Ĥ à¹īà¸Ńà¸ĩ +Ġdé fin +Ġdéfin i +ÙģÙĨ اد +ÙģÙĨاد ÙĤ +à¹Ħà¸Ķà¹ī วà¹Īา +ãģªãģĦ ãĤĪãģĨãģ« +Ġpróp ria +ĠPh át +ãĤĦãģĻ ãģı +สวย à¸ĩาม +ê³ł ìļĶ +Ñı еÑĤ +ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵ ãģĮ +تر جÙħ +ĠкÑĢаÑģ ив +Ġ×ŀ ר×IJש +д еж +ĠÙĬ ÙĪÙĨ +ĠÙĬÙĪÙĨ ÙĬÙĪ +Ñģк оÑĢ +ĠKas ım +ê³Ħ ìķ½ +к оÑģ +Ġна ÑĢÑĥ +ĠнаÑĢÑĥ ÑĪен +Ġdu że +acc ès +Ġh á»ĵng +Ġv Å© +ãģĦãģŁ ãģĹãģ¾ãģĻ +Ġ×ĺ ×Ļ +Ġ×ĺ×Ļ ×ķ׾ +lıkl arı +Ġqu ê +ëħ¸ ëıĻ +ìķ Ķ +CI ÃĵN +Ġt ắc +press ão +ĠìŀĪ ìľ¼ +สิà¸Ĺà¸ĺิ à¹Į +íĥ Ħ +Ġ×Ķ×ŀ ×ŀש׾×Ķ +å¬ī ãģĹãģĦ +ĠÄIJ ặc +ÙĨ زÙĦ +ĠдÑĢÑĥг ой +д ÑĥÑĤ +ìĪ Ļ +Ġth ụ +à¹Ģส ร +à¹Ģสร à¹ĩ +à¹Ģสรà¹ĩ à¸Ī +Ġto plant +Ġtoplant ı +×IJ×ŀ ף +×ķ׾ ת +п омн +Ġyo ÄŁun +ÅĦsk iego +ì° © +ĠØ« ÙĦاث +ĠØ«ÙĦاث Ø© +Ġl ắng +ë¦ ´ +ราà¸Ĭ à¸ģาร +ĠÑģлов а +á» Ĩ +à¸Ķี à¸ģวà¹Īา +ãģĶãģĸ ãģĦãģ¾ãģĻ +Ġд из +Ġдиз айн +fé rence +lıkl ar +ãģªãĤĵ ãģ§ãģĻ +ajÄħ cy +Ġëĭ¤ ìĸij +Ġëĭ¤ìĸij íķľ +ק ×Ļר +ØŃ Ø§Ø± +ส ูà¹ī +Ġz ro +Ġzro bi +Ġzrobi Äĩ +×ŀ ×Ļ׼×Ķ +à¸Ĭà¹Īวย à¹Ģหลืà¸Ń +ĠÑįÑĤ Ñĥ +ë´ ī +楽 ãģĹãģĦ +س ÙĪر +íķĺ ê±°ëĤĺ +Ùħؤ تÙħر +Ġpoc zÄħ +ĠpoczÄħ tk +ĠpoczÄħtk u +Ġع ربÙĬ +اÙĦØ£ ر +اÙĦأر دÙĨ +à¸Ķ ร +Åĵ uvre +ĠÙĪÙĥ اÙĨت +ĠÅĽ redni +Ø® ضر +Ġch uyến +н ÑĤ +ĠìķĮ ê³ł +Ġv á»Ŀi +Ġ×ij ×Ļ×ĵ×Ļ +×ŀ×ĵ ×ķ×ijר +ÙĪ Ùģر +ÙĬ Ø¡ +׳ ×Ľ×¡ +ĠÐĽ а +л он +Ġx ấu +Ùģ ÙĬÙĨ +Ġfé vrier +ĠÐŀ на +ĠV á»ģ +ĠÅŁey ler +ĠполÑĥÑĩ ен +з ад +Ġn ét +à¹Ħà¸Ľ ยัà¸ĩ +×Ĺש×ij ×ķ +à¸ļัà¸Ļ à¸Ĺ +à¸ļัà¸Ļà¸Ĺ ึà¸ģ +Ġgerçek leÅŁ +иÑĩеÑģк ое +ìĪĺ ê°Ģ +Ø« بت +ãģ¤ ãģ¾ãĤĬ +ĠÑĥÑģловиÑı Ñħ +ëĭ¤ ê°Ģ +ราย à¹Ħà¸Ķà¹ī +׼×IJ ×ij +à¹Ĥà¸Ľà¸£ à¹Ĥม +à¹Ĥà¸Ľà¸£à¹Ĥม à¸Ĭัà¹Īà¸Ļ +j ähr +jähr ige +ק ׳×Ļ×Ŀ +×ŀ ×ķק +×ŀ×ķק ×ĵ +ãģ«è¡Į ãģ£ãģ¦ +Ø¢ ÙĦ +вед ение +Ġ׾ ×Ľ×ª×ķ×ij +جÙħ Ùĩ +جÙħÙĩ ÙĪرÙĬØ© +à¸ī à¸ļ +à¸īà¸ļ ัà¸ļ +ĠC òn +à¸ľ สม +ãģªãģ© ãģĮ +×IJ×Ķ ×ij +ĠдейÑģÑĤв иÑı +y ız +à¹Ħมà¹Ī à¹Ģà¸Ħย +ج ÙĪز +×Ķ×Ĺ׾×ĺ ×Ķ +f ällt +ãĥĵ ãĤ¸ +ãĥĵãĤ¸ ãĥį +ãĥĵãĤ¸ãĥį ãĤ¹ +Ġ×IJ ×Ļ׳×Ŀ +ĠнаÑħод иÑĤÑģÑı +Ġdzi ÅĽ +ست Ø·ÙĬع +׾ ×Ļף +Ø® ÙĦاÙģ +Ùĩ ÙIJ +Ġatr ás +íĺ ģ +ãĤĴ ãģĶ +Ġ×Ķ×ŀ ×ķצר +ĠBakan lıģı +ÑİÑī ее +ÙħÙĨ اط +ÙħÙĨاط ÙĤ +Ùģ د +à¸Ļำ à¹Ħà¸Ľ +Ġв аж +Ġваж но +Ġm ạch +׼ ׳×ķ +بع Ø« +lan ması +Ġa yr +Ġayr ıl +ìĤ¬ íļĮ +d ÃŃa +p ÅĤyw +اÙħ ÙĬØ© +íĺ ľ +×IJ׳ ×Ĵ׾ +×IJ׳×Ĵ׾ ×Ļת +ĠìŀĪëĭ¤ ëĬĶ +Ġس اعة +ĠëĤĺ íĥĢ +b ö +à¸Ħ ัà¸Ļ +ĠdziaÅĤ ania +Ø© Ùĭ +Ġng Å© +׳צ ×Ĺ +ãģ¯ ãģĤãĤĭ +ĠyaÅŁ ında +st ück +car acter +caracter ÃŃsticas +Ġr á»Ńa +ĠÙħختÙĦÙģ Ø© +ãģ«ãģĬ ãģijãĤĭ +à¹ģà¸ŀ à¸ĩ +วิ à¹Īà¸ĩ +ת פ×ķ +سا ÙĩÙħ +使 ãģĨ +Ùĥ رÙĬ +×IJ פ×Ļ +........ ....... +ĠÑĤак им +×Ļ׼ ×ķ×Ļ +Ø´ بÙĩ +ج ÙĬر +ãģĿãģ® ãģ¾ãģ¾ +ac jÄĻ +ĠاÙĦت رÙĥ +ĠاÙĦترÙĥ ÙĬ +ĠпÑĢав илÑĮно +Ġت عÙħÙĦ +à¸ģล à¹īา +Ġbi ên +Ġ×ij׳×Ļ ×Ļת +Ġкл Ñĥб +Ġ×ŀ ש×Ķ +в ÑĪий +ãģĵãģ¨ãģĮãģ§ãģį ãĤĭ +à¸ŀัà¸Ļà¸ĺ ุ +à¸ŀัà¸Ļà¸ĺุ à¹Į +ר ×ķ×Ŀ +ĠاÙĦÙģ رÙĨ +ĠاÙĦÙģرÙĨ سÙĬ +à¹Ģà¸Ľà¹ĩà¸Ļ à¸Ħà¸Ļ +ãģĹãģ¦ ãģĬãĤĬ +Ġth ầy +ãĤĵ ãģłãģijãģ© +ìĶ ¨ +Ùħ دÙĨ +ت ÙĪÙĨ +ĠмеÑĤ ал +ĠмеÑĤал л +Ġin ÃŃcio +à¸Ńà¸Ńà¸ģ à¸Īาà¸ģ +ëĴ ¤ +Ġcu á»ijn +Ġbu á»Ļc +ÙĨ سÙĬ +ä cht +×ŀ ×Ļ׳×Ļ×Ŀ +ãģķ ãģ¦ +ãģĮ ãģ§ãģį +ÑĬ ем +Ġtá i +ĠЧ ÑĤ +ĠЧÑĤ обÑĭ +à¸Ľà¸¥ ูà¸ģ +à¸Ĭุม à¸Ĭà¸Ļ +н Ñģкий +Ġv ững +Ġ×Ķ ׾×ij +ë le +Ġש ×¢×ijר +в аÑĤÑĮÑģÑı +б ой +ع ÙĪÙĨ +à¹ģà¸Ķ à¸Ļ +Ġספר ×Ļ×Ŀ +Ġt uyên +Ġnhi êu +ĠQu ý +Ġh uyết +ãĤı ãģĭãĤīãģªãģĦ +Ġ×ŀ ׼ף +Ġ×Ķ ק׾ +Ġ׾×IJ ×ķר +ĠÄIJi á»ĩn +Ø´ ؤ +شؤ ÙĪÙĨ +Ġ×ŀ׊פש +ĠпоÑģÑĤоÑıн но +×ŀ ×Ļר +ìħ Ķ +Ðŀ Ñģ +ÐŀÑģ нов +×ĸ ×Ļת +ĠH á +ĠÑĩаÑģ ов +×IJ ×ķ׾×Ļ +Ġm át +Ø® رÙĪ +خرÙĪ ج +ÙĤ ضا +ÙĤضا ÙĬا +à¹Ģà¸Ľ à¸Ńรà¹Į +ĠÙĬ ÙĪÙĦ +ĠÙĬÙĪÙĦ ÙĬÙĪ +à¹Ĥà¸Ĺ ษ +׳ פ׾ +ת ×ķש +ת×ķש ×ij×Ļ +Ġv ários +×ŀ ר×IJ×Ķ +ëĿ¼ ìĿ´ +ÙĨ غ +×ij צע +г он +ĠÄIJ ược +ع Ùı +пÑĥÑģ к +ĠÙĪاÙĦ Ùģ +üc ü +×Ļק ×Ļ×Ŀ +Ġس بÙĬÙĦ +׾×ij ף +ĠاÙĦÙĤ رÙĨ +ס ×ķת +ĠQu áºŃn +ãģĵãĤĮ ãģĮ +ãĥĸ ãĥ©ãĥ³ãĥī +×Ĵ ×ŀר +Ġwarto ÅĽci +ĠÙĪب ÙĬÙĨ +Ġd ạ +ÐIJ в +ÐIJв ÑĤо +Ġol acaktır +à¸Ļ à¸Ĺà¹Į +Ùħ طار +Ġ×¢ ק×ij +Ġת פ +ãģĹãģ¦ ãģĦãģ¦ +צ ×ŀ×Ĺ +à¸Ī à¸Ńà¸ĩ +Ġö de +ìį ¨ +ÙĨ اس +調 ãģ¹ +ĠогÑĢ омн +ë³´ íĹĺ +×ĺ ק +×ĺק ס×ĺ +ĠbaÅŁ v +ĠbaÅŁv uru +Ġpom ys +Ġpomys ÅĤ +ãģ« ä¹Ĺ +Ġש ׼ף +ĠاÙĦÙħس ؤÙĪÙĦ +Ġз ан +Ġзан ÑıÑĤ +Ġd Æ°Æ¡ng +ãĥĹãĥ¬ ãĤ¤ +ล à¸ļ +ÑĤи ка +ĠAr alık +Ġнед о +Ġm á»Ļ +Ġor an +Ġoran ı +Ġktó r +Ġktór Äħ +Ġ×Ķ×IJ×Ĺר ×ķ׳×ķת +ائ ÙĨ +ÅĦ s +ÅĦs ka +åĽ½ ãģ® +×ŀ ×ĺ×Ļ +ĠвопÑĢоÑģ Ñĭ +à¸Ńà¸ĩà¸Ħà¹Į à¸ģร +×ŀ ×ķצ×IJ +Ġpó ź +Ġpóź niej +ש×ŀ ×IJ׾ +Ġk aps +Ġkaps am +Ġkapsam ında +Ġmá quina +ĠÅĽwie cie +Ġho Ãłng +Ġöz gü +×Ĵ×ķר ×Ŀ +ãģĤ ãģŁãĤĬ +à¸ķัà¸Ķ สิà¸Ļ +à¸ķัà¸Ķสิà¸Ļ à¹ĥà¸Ī +б ÑĢи +ãģ«ãģªãĤĭ ãģ¨ +ت ÙĥÙĪÙĨ +Ġ×ķ×Ķ ×Ļ×IJ +Ġchi ếu +ÑģÑĤан ав +ÑģÑĤанав ли +ÑģÑĤанавли ва +×ŀ ×ķ×Ĵ +c ité +ĠK örper +Ġש ×Ĵ×Ŀ +ع ظ +عظ ÙĬÙħ +Ġ×Ķ×IJ ×Ļש×Ļ +Ġmat ière +ĠÙģ ÙĪÙĤ +Ġk to +Ġkto ÅĽ +à¸Ļ à¹Ĥย +à¸Ļà¹Ĥย à¸ļาย +å¾ħ ãģ¡ +à¹Ģม à¸Ļ +à¹Ģมà¸Ļ ู +A ÃĩÃĥO +Ġt ù +Ġtù y +ãĥĪ ãĥ³ +ĠоÑĤ каз +Ġ×ŀ ×ķצר +ül ü +ãģķãĤĵ ãģ« +Ġ×Ĺ ×ķ×ij +קר ×Ļ×IJ×Ķ +ĠاÙĦØ® دÙħات +ĠÙĦÙħ دة +ر ؤ +رؤ ÙĬØ© +ãĤĴè¦ĭ ãģ¤ãģij +à¸Ł า +Ġréuss i +à¸Ļัà¸ģ à¹Ģรียà¸Ļ +ĠÑĩиÑģ л +à¸ģาร à¹Ģลà¹Īà¸Ļ +Ġhaz ırl +Ġhazırl an +ĠпеÑĢв Ñĭй +ли м +ĠоÑĤзÑĭв Ñĭ +Ġwy jÄħ +ĠwyjÄħ tk +ĠØ£ ÙĤÙĦ +ס ×ļ +Ġê²° ìłķ +Ġ׾×ŀ×¢ ש×Ķ +Ġl ắp +à¹ģà¸ļ ร +à¹ģà¸ļร à¸Ļà¸Ķà¹Į +วà¹Īา à¹Ģà¸Ľà¹ĩà¸Ļ +Ġب دا +Ġبدا ÙĬØ© +ãģ¨ãģĦãģĨ ãģ®ãģĮ +иÑĩеÑģк им +à¸ģาร à¸ŀัà¸Ĵà¸Ļา +Ġb Ãło +Ġmia ÅĤa +y waÄĩ +ĠMär z +ĠÙĨ سبة +Ġéconom ique +×ĸ ×ŀ +×ĸ×ŀ ׳×Ļ×Ŀ +æŃ¢ ãĤģ +Ġt ủ +íķĺ ìĭł +Ġkażde go +stra ÃŁe +à¸Ĭ ีà¹ī +à¹Ģ à¸ļา +ÑĢеÑģ ÑĥÑĢÑģ +ев ой +Ø´ باب +à¸ķà¹Īาà¸ĩ à¸Ľà¸£à¸°à¹Ģà¸Ĺศ +Ġ×IJ ×Ļש +Ġ×IJ×Ļש ×Ļת +×Ļ ×ķפ +×Ļ×ķפ ×Ļ +ĠìļĶ 구 +ì¡° ìĤ¬ +ãģ£ãģŁ ãĤī +׾ ×Ļק +миниÑģÑĤ ÑĢ +ãĤĤãģ® ãģ¯ +Ġl Æ°Æ¡ng +Ġна и +Ġнаи бол +Ġнаибол ее +íİ ĺ +à¹ģà¸ŀ à¹ī +ãĤŃ ãĥ¥ +ĠкоÑĤоÑĢ Ñĭм +à¹ģà¸Ĺ à¸ĩ +à¹ģà¸Ĺà¸ĩ à¸ļà¸Ńล +Ġ׳ ×Ļ×Ķ +Ġ׳×Ļ×Ķ ×ķ׾ +âĤ ª +ĠGi ải +ĠиÑģполÑĮзов а +ëł¥ ìĿĦ +ãģĹãģĭ ãĤĤ +à¸ģà¹ĩ à¸ķà¹īà¸Ńà¸ĩ +ĠÑĢ еб +ĠÑĢеб ен +ĠÑĢебен ка +ت ÙĪاصÙĦ +ãĤ°ãĥ« ãĥ¼ãĥĹ +ãĤĦ ãĤī +à¹Ģà¸Ľà¸´à¸Ķ à¸ķัว +б ÑĢо +ë°ĸ ìĹIJ +ÙĨ Ùİا +×Ķ ×Ĵ +×Ķ×Ĵ ׳×Ķ +à¸Ĺ รั +à¸Ĺรั à¸ŀ +à¸Ĺรัà¸ŀ ยà¹Į +Ġkh á»iji +עצ ×ŀ×ķ +бол езн +Ġë°Ľ ìķĦ +ม à¸Ļ +มà¸Ļ ุ +มà¸Ļุ ษ +มà¸Ļุษ ยà¹Į +âĹ Ĩ +×ŀ צ׾×Ļ×Ĺ +Ñıв ление +Ùħ Ø·ÙĦ +ÙħØ·ÙĦ ÙĪب +Ø® اÙĦÙģ +ت ÙĪÙĤÙģ +ãģ§ãģį ãģ¾ãģĽãĤĵ +оÑģÑĤ ей +м еÑĩа +기 ëĬĶ +תש ×¢ +ص ÙĬب +Ġ×ij×¢ ×ķ×ĵ +à¸Ĥà¸Ńà¸ĩ à¹Ģà¸Ĥา +ÑĤÑı ж +ĠÑĥ пÑĢав +ĠÑĥпÑĢав лениÑı +Ġgén ér +Ġth ÃŃ +פ ×ļ +Ġر Ùħض +ĠرÙħض اÙĨ +Ġtr uyá»ĩn +Ø¥ عداد +ãĤµ ãĥĿãĥ¼ãĥĪ +Ġпол но +Ø® اÙħ +ÐŁ еÑĤ +ÐŁÐµÑĤ еÑĢ +ÐŁÐµÑĤеÑĢ бÑĥÑĢ +ÐŁÐµÑĤеÑĢбÑĥÑĢ г +ÙħÙĨت دÙī +ãģķãĤĮ ãģ¾ãģĹãģŁ +ĠëĮĢ íķĺìŬ +à¸ľà¸¹à¹ī à¸Ĺีà¹Ī +Ġ×ŀ×IJ ×ķ +׾ ׳×ĵ +оÑĩ нÑĭе +ĠнаÑĩ ала +Ġ׾ ×Ļ׾×ĵ×Ļ×Ŀ +ов ое +ãģĻãĤĭãģĵãģ¨ ãģ§ +ĠاÙĦÙĨ Ùģ +ĠاÙĦÙĨÙģ Ø· +ìŀĪ ëĬĶ +غ ÙĨÙĬ +פ ×ĵ +ãĤ ¾ +ĠCr é +ãģ© ãģ¡ãĤī +Ø« اÙĨ +ÑĢаб аÑĤ +ÑĢабаÑĤ Ñĭва +Ġê°Ļ ëĭ¤ +à¸Ī ั +à¸Īั à¸ģร +Ġch ụ +Ġchụ p +Ġм аÑģÑĤ +ĠмаÑģÑĤ еÑĢ +Ġn ắm +ĠÑģÑĤ али +Ġ×Ķ×IJ ×Ļר×ķ×¢ +ãĤ½ ãĥ³ +åĪĨ ãģĭãĤĬ +Ø· بع +بد ا +gr áfico +г еÑĢ +à¸Ķำà¹Ģà¸Ļิà¸Ļ à¸ģาร +Ġsal dır +Ġsaldır ı +в ÑĪиÑħ +ãģĭãģ£ãģŁ ãģ§ãģĻ +Ġyapı yor +ĠاÙĦÙģ ت +צר פת +з доÑĢов +×ij×¢ ׾ +Ġ×IJ ×ŀ×Ļת×Ļ +Ġоб Ñĭ +ĠобÑĭ Ñĩ +ĠобÑĭÑĩ но +Ġ׾ ×ķ×ŀר +ت ÙĥÙĨ +تÙĥÙĨ ÙĪÙĦÙĪج +تÙĥÙĨÙĪÙĦÙĪج ÙĬا +Ġhakk ı +ĠÑĢаР² +ĠÑĢав но +رÙĬ Ùĥ +Ġ×ij ×ŀ×Ļ×ĵ +Ġ×ij×ŀ×Ļ×ĵ ×Ķ +à¹ģà¸ģ à¹īว +Ġìĸ ĺ +Ġìĸĺ 기 +ãģĹãģ¦ ãģĦãģ¾ãģĹãģŁ +Ġkı sm +Ġkısm ı +ê± ¸ +åĨħ ãģ® +ì§ ķ +à¹Ģหมืà¸Ńà¸Ļ à¸ģัà¸Ļ +ĠÙģ ÙIJ +ĠÙģÙIJ ÙĬ +ÙĤ اعدة +Ġmoż esz +Ùħ صاÙĦ +ÙħصاÙĦ ØŃ +ãģ¾ãģŁ ãģ¯ +б ег +Ġs ıc +Ġsıc ak +Ñĩ иÑģ +ÑĩиÑģ лен +Ġн ог +ãĥģãĥ£ ãĥ³ +ãĥ« ãĥī +Ġgi ó +Ġs ını +Ġsını f +ив аÑĤÑĮ +Ġqu ên +Ġì łģ +Ġìłģ ìļ© +ĠJo ão +Ùģ اد +ĠGl ück +à¸Ĺ à¸Ńà¸Ķ +Ġg ói +ï¼ Ĭ +Ġdé tail +ĠدÙĬ سÙħ +ĠدÙĬسÙħ بر +ë¡ľ ìĦľ +×ŀ ×ķ×Ĺ +à¹Ħ ฮ +ĠоÑĤ д +ĠоÑĤд ÑĭÑħ +Ġkh uyến +à¸Ħ à¸Ńย +Ġج ÙĨÙĬ +ĠجÙĨÙĬ Ùĩ +ĠاÙĦد Ùģاع +à¸Ļà¹īำ หà¸Ļัà¸ģ +ĠìĤ¬ëŀĮ ëĵ¤ìĿ´ +Ġth ừa +ĠÃ¶ÄŁrenc i +ĠпомоÑī и +ĠczÄĻ ÅĽÄĩ +ש ×ĺר +ĠN hi +ĠNhi á»ģu +׳ צ×Ļ +ĠнаÑĪ ем +ĠkarÅŁÄ± laÅŁ +Ġ×Ķש ׳×Ļ×Ŀ +ĠÄIJ Æ°á»Ŀng +Ġtr ú +ĠÑĢазлиÑĩ нÑĭÑħ +ĠاÙĦØ´ Ùĩر +Ġ×ľ×¢ ×ķ׾×Ŀ +ØŃ Ø¬Ø± +ĠÄij á»ķ +ĠìĿĺ íķ´ +à¸ļ à¹Īà¸Ńย +Ġ×Ķ ×Ļ׾×ĵ +ãģ¨ãģª ãģ£ãģŁ +Ġ×Ĺ×ķ ×ķת +Ġש×Ļר×ķת ×Ļ +Äħ cy +س رÙĬ +K Ä° +פ ׳×ķ +ÑģÑĤÑĢÑĥк ÑĤÑĥÑĢ +ÑĤ ÑĢÑĥд +Ġ×Ķ קר +Ġ×Ķקר ×ķ×ij +Ġth áºŃm +èģŀ ãģį +ÙĤÙĪ ÙĬ +клÑİÑĩ ен +ÑĤе Ñħ +ÑĤеÑħ нолог +è¡Į ãģ£ãģŁ +Ġ×ķ×IJ ×Ļף +ĠÅŁek lin +ĠÅŁeklin de +r ô +ÑĢ ог +Ġнов Ñĭе +Ġס ×ij×Ļ×ij +Ġtecn ologÃŃa +ס ׼ +×¡×Ľ ×ķ×Ŀ +ĠÅŀ ub +ĠÅŀub at +Ġ×Ķ×ŀ ׾×IJ +Ġwy pos +Ġwypos aż +ãģ¯ ä½ķ +ãĤ¬ ãĥ³ +ê° ĸ +Ġкак ие +Ġçocuk lar +Ġ׾צ ×ĵ +Ġkay ıt +ĠмеÑģÑĤ е +Ùħ دÙĬÙĨØ© +Ġ׼ ×Ĵ +Ġ׼×Ĵ ×ķף +ãģĹãģ¦ ãĤĭ +ĠÙħا ÙĬÙĪ +ãģ£ãģ¦ãģĹãģ¾ ãģ£ãģŁ +ĠпÑĢогÑĢамм Ñĭ +à¹ģล à¸Ļà¸Ķà¹Į +ãĥ¯ ãĤ¤ +ער ×ķ×¥ +Ñģ ид +ĠB öyle +Ġì²ĺ ìĿĮ +Ġת פק×Ļ×ĵ +ĠTr ên +íĥ Ī +ĠÐłÐ¾ÑģÑģ ий +ĠÐłÐ¾ÑģÑģий Ñģкой +Ġs Ãłn +Ġrè gle +ĠyaklaÅŁ ık +à¹Ģล ิà¸ģ +Ġد ائÙħ +Ġ×ķ ×Ĵ +اب ر +Ġb è +ĠاÙĦ ÙĤدÙħ +ĠÑĢеÑĪ ениÑı +hi ên +ÑĤи к +Ä Ħ +à¸ļรร ยาà¸ģ +à¸ļรรยาà¸ģ าศ +רצ ×ķף +åĭķ ãģį +ĠGä ste +Ġ기 본 +ĠÙĬ عرÙģ +ĠS á»Ń +gÅĤ ÄĻb +à¹Ģà¸Ń ส +×IJ×ŀ ×Ļף +Ġп Ñĥнк +ĠпÑĥнк ÑĤ +Ġ×Ļ×ķ×ĵ ×¢×Ļ×Ŀ +ãĤ« ãĥ©ãĥ¼ +Ġ×ijס ×ĵר +Ġbu á»ĵn +й ÑĤ +йÑĤ еÑģÑĮ +ãĤĴ æ±ĤãĤģ +Ġ×IJת ׼×Ŀ +Ġ모 르 +ظ رÙĪÙģ +Ñĩ еÑģÑĤво +ìĸ´ ìĦľ +Ġод на +Ġkap ı +Ġëħ¸ ëł¥ +ĠKü che +ĠاÙĦت Ø´ +Ø· ÙĬب +ĠíĬ¹ íŀĪ +ĠвÑĭп ÑĥÑģ +ĠвÑĭпÑĥÑģ к +×ĵ ת×Ļ +Ġu ÄŁ +ĠuÄŁ ra +ائ Ùĩا +Ġtho át +ãģª ãĤĤãģ® +Ñij ÑĢ +기 ê°Ģ +ĠgeliÅŁ me +تØŃ ÙĤ +تØŃÙĤ ÙĤ +Ġоп аÑģ +б ÑĢоÑģ +ห ุ +หุ à¹īà¸Ļ +ì¼ Ģ +ãĤ¹ ãĥŀ +ãĤ¹ãĥŀ ãĥĽ +Ø£ Ùģر +Ø£Ùģر اد +ĠTh á»±c +Ġth ắ +ãĥªãĥ³ ãĤ¯ +Ġni á»ģm +ĠHö he +عÙħ ار +ÙĥÙĪر ÙĪÙĨ +ÙĥÙĪرÙĪÙĨ ا +ĠÄIJ ến +ĠÑģам ом +ĠÑĤ еле +ĠÄijo án +à¸Ħวามà¸Ħิà¸Ķ à¹Ģหà¹ĩà¸Ļ +Ġд иÑģк +Ø£ Ø·ÙģاÙĦ +ม ารà¹Į +à¸Ĺ หาร +à¸Ĺ à¸Ļ +Ġب عÙĬد +ĠاÙĦÙĩ ÙĨد +åĩº ãģĹãģ¦ +Ġkar de +Ġkarde ÅŁ +×Ķ×Ļס×ĺ ×ķר +×Ķ×Ļס×ĺ×ķר ×Ļ×Ķ +éģ¸ ãģ³ +ع اÙħÙĦ +à¸Ĥ ยาย +Ġtü rl +Ġtürl ü +ĠìĿ¼ ìĿ´ +Ġmaté ria +Ġ׼׾ ×ķ×ŀר +ãĥģãĥ£ ãĥ¼ +جÙħ اعة +ĠÑģво им +Ø¥ÙĤ اÙħØ© +ä¾ĭ ãģĪãģ° +س اب +Ø¢ خر +ÙĤ دÙĬر +×IJ×ŀ ×Ļ +ìĸ » +Ġ׳×ķס פת +ĠÐĴ лад +ĠÐĴлад им +ĠÐĴладим иÑĢ +Ġest ará +ãģĵãģĨ ãģĦãģĨ +ãĤĴ 使çĶ¨ +มา à¸ķร +มาà¸ķร à¸IJาà¸Ļ +ãģ£ãģ ½ +Ġn ú +Ġnú i +ย าà¸ĩ +ĠاÙĦج ÙĨس +Ġüst ün +ëľ » +ãĤ» ãĥ« +ãģ¦ãģĦ ãģįãģ¾ãģĻ +Ġ×Ĺ ×ķ×ĸ +Ġ×Ĺ×ķ×ĸ ר +ĠÐĵ лав +à¹Ĥà¸Ĭ à¸Ħ +íı IJ +ÙĨت ظر +Ġ×Ĵ ×ij×Ļ +ع ÙĤب +int ér +intér êt +×ŀ פ×Ĵ +×ŀפ×Ĵ ש +Ġth ù +اÙģ ت +Ġ×ŀש פ +Ġ×ŀשפ ×ĺ×Ļ +ĠÙħ ÙĪاÙĤع +è¦ ļ +è¦ļ ãģĪ +×ĵ ×Ļף +à¹Ģรืà¹Īà¸Ńà¸ĩ ราว +ãģ¾ ãģĤ +Ġgh ế +иÑĢÑĥ ÑİÑĤ +à¸ģ ว +à¸ģว à¹īาà¸ĩ +Ġпов еÑĢ +ĠповеÑĢ Ñħ +ĠповеÑĢÑħ ноÑģÑĤ +׳ ×ĵר +Ġкон ÑĨе +Ġдолж на +Ġ×Ļש ×Ļר +acaģı z +ìĹ Ķ +Ġn ÃŃvel +Ġö r +Ġör nek +Ùĥ Ùģ +ĠФедеÑĢ аÑĨии +Ġ구 ìĦ± +หัว à¹ĥà¸Ī +ĠV áºŃy +м ед +мед и +меди ÑĨин +медиÑĨин Ñģк +از ÙĬ +×Ĵ×ij ×ķ׾ +ÑĦ ÑĢ +Ġzus ätzlich +à¸ģ à¸ģ +ĠاÙĦاÙĤتصاد ÙĬØ© +Ġh è +lu ÄŁun +ج Ùİ +à¹Ħà¸Ł ลà¹Į +ÄIJ T +ãģĿãģ® ä»ĸ +à¸Ĺิ à¹īà¸ĩ +ĠاÙĦØ£ ÙĪ +ر سÙħ +æ°Ĺ ãģ¥ +ìĿ´ ë©° +ÑĮ ев +ص Ø· +ĠاÙĦاست Ø« +ĠاÙĦاستث Ùħار +à¸Ńา à¸Ħาร +ĠÑĤоÑĩ но +ĠV ân +à¸Ń ร +à¸Ńร à¹Īà¸Ńย +ĠاÙĦس ÙĨØ© +Ġc Æ°á»Ľi +×Ļ×Ķ ף +íį ¼ +話 ãģĹ +âĹ ĭ +ĠìķĬ ìĿĢ +ãĥ¡ ãĥ¼ãĤ +ãĥ¡ãĥ¼ãĤ « +ãĥ¡ãĥ¼ãĤ« ãĥ¼ +ĠÑĤеп ло +å½¼ ãĤī +ĠÄ° z +ĠÄ°z mir +íĻ į +Ġr ượ +Ġrượ u +æĢĿãģĦ åĩº +ĠPh ạm +Ġchá u +צ×Ļ ×ķת +ĠìĿ¼ 본 +ìĤ¬ ëĬĶ +ĠÑģозд ан +Ġar acı +Ġ×¢ ר +Ġער ×Ļ׼×Ķ +ĠíķĺëĤĺëĭĺ ìĿĺ +dzi ÅĤ +à¸Ľà¸£à¸° à¸ĺาà¸Ļ +Ġser ÃŃa +ĠìŀĪ ëıĦë¡Ŀ +در ج +íķľëĭ¤ ëĬĶ +à¸Ńา à¸Ĺ +à¸Ńาà¸Ĺ ิà¸ķ +à¸Ńาà¸Ĺิà¸ķ ยà¹Į +ÑĤелÑĮ нÑĭй +ĠØ® دÙħات +×ŀ׳ ×ĺ +Ġl ược +ĠS Ãłi +ĠÙĪ اض +ĠÙĪاض ØŃ +غ از +ĠdoÄŁ al +Ġ×ijש ×Ŀ +Ġд лин +ĠØ¥ طار +Ġ×ijס פר +ãĤĴ ä¸İ +ãĤĴä¸İ ãģĪ +Ġë²ķ ë¥ł +ĠÑĥ вели +ĠÑĥвели Ñĩи +ส à¹Ħà¸ķ +สà¹Ħà¸ķ ลà¹Į +à¹Ħ à¸ģล +×ij׊ף +ĠìĿ´ íĽĦ +Ġm unic +Ġmunic ÃŃpio +تÙħ Ø«ÙĦ +ĠÄij áo +H ôtel +Ġl á»Ńa +ĠÄij ẳng +Ñĩ ки +Ø´ رÙĪ +شرÙĪ Ø· +ĠìĿ´ 를 +ÙĬ Ùĭا +×ŀ׾ ×ļ +×ŀ×Ķ ×Ļר×ķת +ĠобÑıз аÑĤелÑĮ +ĠобÑıзаÑĤелÑĮ но +é nergie +Ġmud ança +Ġm ụ +Ġmụ n +Ġn º +ĠاÙĦت عا +ĠاÙĦتعا ÙĪÙĨ +ĠاÙĦاجتÙħاع ÙĬØ© +Ġп лаÑģÑĤ +Ġëĵ± ìĿĺ +ãĥIJãĤ¤ ãĤ¯ +Ùĩج ÙĪÙħ +ĠSa úde +Ġì¤ijìļĶ íķľ +Ġ×Ķצ ×Ļ×ij×ķר +תק ף +ĠاÙĦعاÙĦÙħ ÙĬ +ĠболÑĮÑĪ ой +ĠÙĥ ÙĦÙħ +ĠÙĥÙĦÙħ Ø© +ãģ®ãģ§ãģ¯ãģªãģĦ ãģ§ãģĹãĤĩãģĨãģĭ +ĠÙħ باراة +Ġש×IJ ׳ +Ġש×IJ׳ ×Ĺ׳×ķ +ãĤ¹ãĤ¿ ãĤ¤ãĥ« +ĠSa ÄŁ +ĠSaÄŁ lık +Ġh Æ° +׳ ×Ĺ×Ķ +Ġ×ij קר×ij +Ø· عÙħ +ห ิà¸Ļ +à¸Ĺุà¸ģ วัà¸Ļ +à¸Ħรัà¹īà¸ĩ à¸Ĺีà¹Ī +ĠlÃł nh +Ġdonn é +ãģĽ ãģĦ +جز ÙĬرة +доÑĢ ож +ì¼ ľ +تÙĨظ ÙĬÙģ +ãĥģ ãĥ§ +Ġald ıģı +ج اج +ĠÑĤ омÑĥ +à¸Ľ ิ +Ġ×ijר שת +ãģıãģªãĤĬ ãģ¾ãģĻ +ĠпÑĢин ÑĨип +Ġ׊׾×ķ +ëı ¼ +×ķ×Ĵ ש +س س +à¸Ľ ู +Ġh ầu +æĦŁãģĺ ãĤĭ +ï¼ ´ +د ÙĪا +ĠÑģм ог +scri ção +Ġth áºŃn +Ġר ×ķ×IJ×Ķ +обÑĢаж ен +ĠاÙĦتج ارÙĬØ© +Ø· بÙĬع +jÄħc Äħ +íĸī ìľĦ +Ġнов Ñĭй +Ġ×ŀ ×Ĺ×ĵש +æĮ¯ ãĤĬ +gu é +Ġ×IJ ×Ļר×ķ×¢ +Ġ×IJ×Ļר×ķ×¢ ×Ļ×Ŀ +ĠاÙĦ Ø°Ùĩب +×ĵ ×IJ +ت اÙĨ +ãģł ãģĹ +à¸Ńั à¸ķรา +à¹Ĥ à¸Ī +بÙĦ اد +×Ķ×Ļ ×Ļ׳×ķ +ĠÑģп е +ĠÑģпе ÑĨиалÑĮно +ĠÅĽwi ata +ãĤĵãģ§ãģĻ ãĤĪ +شر ÙĥØ© +ĠpÅĤ yt +Ġsitu é +Ġ׼×IJ ׾×Ķ +ס ×ijר +Ġkaż d +Ġkażd ym +ãĤĴæĮģ ãģ¤ +׾×Ķ ׾ +׾×Ķ׾ ף +ĠwÅĤ as +ĠwÅĤas ne +ĠsaÄŁ lan +×ŀ×¢ ׾×Ķ +ĠاÙĦا ÙĪÙĦ +ìĹIJìĦľ ëıĦ +×IJ×Ļר ×ķפ×Ķ +تÙĤ ÙĨÙĬØ© +Ùħ ائ +Ùħائ Ø© +Ġcompañ ÃŃa +Ġsü rek +Ġsürek li +ĠиÑģ кÑĥÑģ +ĠиÑģкÑĥÑģ ÑģÑĤв +ĠB ürger +ת ×Ĺר +ת×Ĺר ×ķת +à¸ŀรà¹īà¸Ńม à¸ģัà¸ļ +Ø´ Ùħ +à¸ĸืà¸Ń วà¹Īา +è¾¼ ãĤĢ +ä¼ij ãģ¿ +ĠاÙĦØ£ ب +ĠÑģÑĤоим оÑģÑĤÑĮ +ĠпÑĢав а +may ın +ห วย +ĠاÙĦØ· بÙĬعÙĬ +à¸Ĺีà¹Ī à¸ŀัà¸ģ +ĠEst á +Ñĭва ÑİÑĤ +ب سÙĬ +بسÙĬ Ø· +Ġ×ij×¢ ×ijר +åı¯èĥ½ ãģ§ãģĻ +Ġ×ĵ ×ķ׾ +Ġ×ĵ×ķ׾ ר +Ùĩ Ùİا +воÑĢ оÑĤ +ãģ¦ ãģĦãģ¾ãģĹãģŁ +à¹Ĥà¸Ĺร ศ +à¹Ĥà¸Ĺรศ ั +à¹Ĥà¸Ĺรศั à¸ŀ +à¹Ĥà¸Ĺรศัà¸ŀ à¸Ĺà¹Į +Ġק ׳ +ĠاÙĦØ« ÙĨ +ĠاÙĦØ«ÙĨ ائÙĬØ© +Ġco ût +à¸ķิà¸Ķ à¸ķัà¹īà¸ĩ +Ġö rg +Ġörg üt +ĠاÙĦØ® ÙĦÙĬ +ĠاÙĦØ®ÙĦÙĬ ج +Ġb á»įn +×ķ׾×ķ×Ĵ ×Ļ +ëŀ ľ +ĠÐij олÑĮ +ĠÐijолÑĮ ÑĪ +×Ĵ ×ijר×Ļ×Ŀ +ÙĤ ÙĬد +×ij×Ļ×ĺ ×ķ×Ļ +æīĵ ãģ¡ +Ġol muÅŁ +f äh +fäh ig +ล าà¸Ļ +ĠÙĤ طر +ש פ×Ķ +èªŃ ãĤĵãģ§ +à¸Ĥ วา +Ġchi ếm +ãĤ¤ãĥ³ ãĤ¿ +ãĤ¤ãĥ³ãĤ¿ ãĥ¼ãĥ +ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥ į +ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥį ãĥĥãĥĪ +Ġ׾ש×ŀ ×ķר +Ġت رÙĥ +ĠترÙĥ ÙĬا +ר ×ķ×ĺ +ãģ¨æĢĿ ãģĦãģ¾ãģĹãģŁ +ĠاÙĦت ÙĤ +Ġd Æ° +ãģ¦ãģıãĤĮ ãĤĭ +ãģĹãģŁ ãģĵãģ¨ +Ġróż ne +ĠاÙĦØ· ÙģÙĦ +ĠPost é +Ġ×ŀש ×ķ×Ŀ +Ñį ÑĢ +ĠÑĢабоÑĤ аеÑĤ +ãĤ· ãĥª +ãĤ·ãĥª ãĥ¼ãĤº +Ġ×ij×Ķ ×Ĺ׾×ĺ +ק×Ķ ×Ļ׾×Ķ +ãĤ« ãĥ¡ +ãĤ«ãĥ¡ ãĥ© +ï¼ ¯ +ĠìĤ¬ ìĿ´ +Ġk ì +Ġth Æ°á»Ľc +ض بط +ÙĤب ÙĪÙĦ +åĪ¥ ãģ® +Ġparticul ière +ĠÑģво ем +Ġ×¢ סק +Ġעסק ×Ļ×Ŀ +×ij×Ĺ ×Ļר×ķת +×ij ×Ļ׳×ķ +à¸ĭ à¸Ń +Ġ×¢ ×ķ×ijר +ãģłãģ£ãģŁ ãģ®ãģ§ +ıld ıģı +Ùħ دار +Ùħدار س +주 ìĭľ +à¸Ńา ศ +à¸Ńาศ ัย +Ġt ấm +à¸ŀิ à¸Ī +à¸ŀิà¸Ī าร +à¸ŀิà¸Īาร à¸ĵา +ÑĤелÑĮ нÑĭе +Ñģк ÑĥÑİ +Ðľ Ðĺ +à¹Ģà¸ģ า +à¹Ģà¸ģา หล +à¹Ģà¸ģาหล ี +×ĵ ×Ĺ +à¹Ģà¸Ĭ ิà¸ĩ +Ġد ÙĤÙĬÙĤØ© +íķĻ ìĥĿ +Ġש×IJ ׾×Ķ +Ġcontr ôle +Ġsit uação +à¸Ĥà¸Ńà¸ĩ à¸ľà¸¹à¹ī +ÙĨ Ø·ÙĤ +ê³¼ íķĻ +หลาย à¸Ħà¸Ļ +Ġn ắng +ÙĤ Ùı +ì¡° ê±´ +Ñ ķ +ãĥĥ ãģ¨ +×ŀ ×Ļ׾×Ķ +Gr ün +×Ļ ×Ļ×¢ +×Ļ×Ļ×¢ ×ķ×¥ +×ŀ׳ ׼ +ë ŃIJ +×ŀ×¢ ×ŀ×ĵ +สำ à¸Ļัà¸ģ +ج دد +à¸Ħ ัà¸Ķ +Ġ×Ķ×ŀש פ +Ġ×Ķ×ŀשפ ×Ĺ×Ķ +×ŀש ק׾ +ÙĦ Ùı +Ġty tu +Ġtytu ÅĤ +ÑĪ ей +ĠìĿ¼ ë¶Ģ +ÑĪ ение +Ġph óng +ĠìĹŃ ìĤ¬ +ãĤ« ãĥ³ +Ġtú i +ĠÙĨ ÙĪÙģ +ĠÙĨÙĪÙģ Ùħبر +gr ün +ĠاÙĦØ´ ÙħاÙĦ +ÅĽwi adc +ÅĽwiadc zenie +ער ×Ķ +Ġ×¢ ×ķ×ij +Ġ×¢×ķ×ij ×ĵ×Ļ×Ŀ +×ĵ×ķ×Ĵ ×ŀ×IJ +ä»Ĭ ãģ¯ +Ġv ão +ĠТ ем +Ñģ илÑĮ +Ġch ợ +Ùħ را +Ùħرا ÙĤب +à¹Ħมà¹Ī รูà¹ī +Ġر ائع +×IJ׳ ×Ĺ׳×ķ +สà¹Īà¸ĩ à¹Ģสริม +צ ×Ĺ +ĠìŀĪìĸ´ ìĦľ +Ġkur ulu +Ġkurulu ÅŁ +ĠÃĸ zellik +ĠÃĸzellik le +Ġת ×Ļק +Ġgh é +Ġspr zÄĻ +ĠsprzÄĻ t +ער ×ķת +را ØŃØ© +ãģ£ ãģį +ãģ£ãģį ãĤĬ +ĠìķĦ ëŀĺ +stit uição +Ġдолж но +×Ķ רש +×Ķרש ×ŀ×Ķ +×Ķ׾ ×ļ +ãģ¡ ãģª +ãģ¡ãģª ãģ¿ +ãģ¡ãģªãģ¿ ãģ« +פ ×Ĺ×ĵ +ĠاÙĦج ÙħÙĬع +×ij×¢ ׾×Ļ +Ġtr ùng +Ġפ ת×Ĺ +×ŀ׾×Ĺ ×ŀת +ãĥĨ ãĥ¼ãĥ +ãĥĨãĥ¼ãĥ ŀ +Ùħ تاب +Ùħتاب عة +Ġ모 ìĬµ +ÙĬ ص +åIJĪ ãģĨ +ĠY ap +ĠYap ı +ĠÑģ казаÑĤÑĮ +ëª ° +à¸Ĺีà¹Ī สำà¸Ħัà¸į +ĠìĹĨ ìĬµëĭĪëĭ¤ +Ġnh ắc +Ġülk eler +Ġмног ие +íķĺ ìħ¨ +มาà¸ģ à¸Ĺีà¹Īสุà¸Ķ +à¸ģ à¹īา +à¸ģà¹īา ว +ĠÄ° yi +л еж +леж а +ãĤ¸ ãĥ§ +à¸Ĺั à¸ŀ +ا ÙĪر +Ġ×Ĺ×ijר ×Ļ +Ġ׾ ש×Ŀ +ì² « +ĠT á»Ń +×ŀ ×ķ׳×Ļ +ÙĤ ÙĪد +à¸ģระ à¹Ģà¸Ľ +à¸ģระà¹Ģà¸Ľ à¹ĭ +à¸ģระà¹Ģà¸Ľà¹ĭ า +ĠпÑĢоблем Ñĭ +Ġaç ıs +Ġaçıs ından +Ġ×Ķ×ŀ ׼ +ĠÙħع ظÙħ +ÙĤÙĬ اس +ĠпÑĢод олж +ĠпÑĢодолж а +Ġver diÄŁi +ĠпÑĢед меÑĤ +ãģĦãģ¾ãģĻ ãģĮ +ĠëĶ° 른 +ĠاÙĦ ÙĤÙĬاÙħ +ĠØ¥ÙĦÙĬ Ùĩا +Т ÐIJ +п оз +ãĤ· ãĥ¥ +ä¸ĬãģĮ ãĤĬ +à¹Ģà¸Ķิม à¸ŀัà¸Ļ +à¸ģุ ล +ØŃر ÙĬØ© +ק×ij×ķצ ×ķת +ë¯ ¿ +ĠاÙĦÙħ ÙĨا +ĠاÙĦÙħÙĨا Ø·ÙĤ +ĠвÑĭп ол +ĠвÑĭпол нÑı +ãĥĭ ãĤ¢ +Ġê²° êµŃ +×Ĺ ×ķ×ŀ +×Ĺ×ķ×ŀ ר×Ļ×Ŀ +ĠУкÑĢа инÑĭ +ห à¸Ńม +ר ×Ļס +ĠÑħоÑĤ ел +ĠобÑĢаз ованиÑı +Ġkh ẳng +Ġm Æ°a +Ġgör me +Ġgüç lü +سع Ùī +มัà¹Īà¸Ļ à¹ĥà¸Ī +íķĺ ê²łìĬµëĭĪëĭ¤ +Ġпол Ñĥ +Ġfün f +ãģ¨æĢĿ ãģ£ãģ¦ãģĦãģ¾ãģĻ +Ġê·¸ê²ĥ ìĿĢ +ĠdÃ¼ÅŁÃ¼n ce +ìŀ ł +ĠH Æ°á»Ľng +ĠTi á»ĥu +Ġç ift +ãģij ãģ° +à¸Īà¸Ļ à¸ĸึà¸ĩ +à¸Ĺำ à¹Ħà¸Ķà¹ī +ĠìŀIJ ì²´ +Ġd õ +Ġdõ i +à¸Ī ัà¸Ļ +à¸Īัà¸Ļ à¸Ĺ +à¸Īัà¸Ļà¸Ĺ รà¹Į +ece ÄŁini +׳×ķ×¢ ר +غ ار +ĠاÙĦØ£ÙħرÙĬ ÙĥÙĬ +داع Ø´ +ĠбезопаÑģ ноÑģÑĤи +Ġб Ñİ +ĠбÑİ Ð´Ð¶ +ĠбÑİдж еÑĤ +ãĥĬ ãĤ¤ +à¸ŀà¸ļ วà¹Īา +da ÄŁ +×IJ ×ķפף +íĹ Į +ãĥĢãĤ¤ ãĤ¨ +ãĥĢãĤ¤ãĤ¨ ãĥĥãĥĪ +ĠëĮĢ íĨµ +ĠëĮĢíĨµ ëł¹ +D Ä° +Ø£ ØŃداث +ĠA ÄŁ +ĠAÄŁ ust +ĠAÄŁust os +ØŃÙĦ ÙĪÙĦ +Ġw ÅĽ +ĠwÅĽ ród +ĠÑģо оÑĤвеÑĤ +ĠÑģооÑĤвеÑĤ ÑģÑĤв +ĠÑģооÑĤвеÑĤÑģÑĤв ии +ĠLu áºŃt +Ġ׼׾ פ×Ļ +Ġв еÑī +ĠвеÑī еÑģÑĤв +ק ×Ļ×¥ +ĠبÙĩ ذا +عا Ø´ +à¹Ģà¸Ľà¹ĩà¸Ļ à¹Ģรืà¹Īà¸Ńà¸ĩ +Т Ðķ +Ġ×ij×IJ ×Ļ׳×ĺר׳×ĺ +س عد +Ġ×Ķ×ĺ ×Ļפ×ķ׾ +פ ×Ļס +à¸ĩà¹Īาย à¹Ĩ +ĠGer ät +׾ ×Ļ×ĵ×Ķ +ĠÑĢ иÑģк +׾ק ×Ĺ +н наÑı +ר ×Ļ×ĵ +п ÑĢакÑĤи +пÑĢакÑĤи к +à¸Ĥัà¹īà¸Ļ à¸ķà¸Ńà¸Ļ +à¸Ļà¹Īา รัà¸ģ +larınız ı +à¸Ńà¸Ļุ à¸įา +à¸Ńà¸Ļุà¸įา à¸ķ +ĠzdjÄĻ cia +Ġb ây +Ñģ ÑĢ +ÑģÑĢ оÑĩ +ãĥĭ ãĥ³ãĤ° +Ġö ner +Ġöner i +Ġнов ÑĭÑħ +دع ÙĪØ© +Ġg ắn +ĠاÙĦÙĦ بÙĨ +ĠاÙĦÙĦبÙĨ اÙĨÙĬ +ãĥĨãĤ£ ãĥ¼ +Ġص ØŃÙĬØŃ +ем ÑĭÑħ +çĸ² ãĤĮ +ĠпÑĢо иÑģ +ĠпÑĢоиÑģ ÑħодиÑĤ +ส à¸ķิ +ĠT ết +Ġ×Ķ׾ ׾×ķ +à¹Ģรืà¹Īà¸Ńà¸ĩ à¸Ļีà¹ī +×ŀ×ij ׳×Ķ +Ġconte údo +Ġا خت +Ġاخت ÙĬار +Ùħ سÙĦ +ÙħسÙĦ سÙĦ +ëı Ī +Ġ׾ ×Ļ×ĵ +à¸ŀิ à¸ĺี +ĠÑģов Ñģ +ĠÑģовÑģ ем +ãģĮãģĤãĤĬ ãģ¾ãģĹãģŁ +Ġsó ng +Ø¥ صÙĦاØŃ +ë§ ģ +Ùģ ÙĬر +ĠJe żeli +ìłľ ëıĦ +d ÅĤug +ìĥģ ìĿĦ +Ġc áºŃn +Ġhá»į p +Ø£ ست +أست اذ +Ġ×ŀ ×Ļש×Ķ +Ġ×ŀ×Ļש×Ķ ×ķ +Ġd Ãły +Ġch Ãłng +ãģ¡ãĤĥãĤĵ ãģ¨ +ĠÄij ám +Ġsw ój +Ġpoder á +ĠоÑĤлиÑĩ а +Ġpéri ode +ünd ig +×ĺ×¢ ף +ÑģÑĤÑĢо иÑĤелÑĮ +ר ת×Ļ +Ġ×Ļ×Ķ ×Ļ×ķ +׾ ס +ĠاÙĦÙħÙĨ زÙĦ +à¸Ļิ à¹īว +иÑĦ ика +иÑĦика ÑĨи +ðŁĺ ī +Ġad ına +ãĢĤãĢĤ ãĢĤ +×IJ ×Ļף +ס ×Ļר +ĠÙĬ عد +çŃĶ ãģĪ +اÙĦ جز +اÙĦجز ائر +енÑĮ к +ร ห +รห ัส +ĠTürk çe +ê¾ ¸ +Ġ×Ļ ×ķ׼׾ +Ġש ×ķ׳×Ķ +Ġ×ij×ŀ צ×ij +ĠдейÑģÑĤв иÑĤелÑĮно +ĠبأÙĨ Ùĩ +×ŀק ×ĵ +Ġ×Ķש ק +Ø®ÙĬ ارات +Ġf ı +Ġfı rs +Ġfırs at +ëij ĺ +ĠìĦľ ìļ¸ +Ġ×Ķ×Ĵ ×ķ×£ +ر عا +رعا ÙĬØ© +ĠK ết +к Ñģи +ĠÑĥÑģлÑĥг и +ноÑģÑĤ ей +ìļ´ ëıĻ +ĠобÑĬ Ñı +ĠобÑĬÑı вл +н еж +×Ķפ ×ļ +Ġ×ij×¢ ×Ļ׳×Ļ +ëĨ Ĵ +ĠпÑĢоÑĨ ед +ĠпÑĢоÑĨед ÑĥÑĢ +Ġiht iy +Ġihtiy acı +Ġë°Ķ ëŀį +Ġë°Ķëŀį ëĭĪëĭ¤ +à¸ģล ัว +ĠÑģл ожно +ק×Ļ ×Ļ×ŀת +ĠÄIJ ình +ĠÙħ ÙĦÙģ +Ġà¹Ĥà¸Ķย มี +Ġkat kı +تØŃ ÙĪÙĬÙĦ +à¹Ħ à¸ŀ +ĠH á»į +ñ e +Ġдо Ñħод +Ġtho ải +íķĺìŬ ìķ¼ +ãĤ¹ãĥĿ ãĥ¼ãĥ +ãĤ¹ãĥĿãĥ¼ãĥ Ħ +ĠG òn +Ġk è +Ġkè m +éĢ² ãĤģ +ãĤ¹ ãĥ¼ãĥ +ãĤ¹ãĥ¼ãĥ ij +ãĤ¹ãĥ¼ãĥij ãĥ¼ +ĠgiÃł u +ĠØ¥ عادة +Ġ׾ ×ķק +Ġ׾×ķק ×Ĺ +ĠÑħоÑĩ еÑĤ +×ĺ ׾×ķ×ķ +×ĺ׾×ķ×ķ ×Ļ×ĸ +×ĺ׾×ķ×ķ×Ļ×ĸ ×Ļ×Ķ +Ġth uyết +ãģĿãĤĮ ãģ§ +Ġvard ı +à¹Ħร à¹ī +ع بد +ĠRep ública +ãĥ¼ãĤ¿ ãĥ¼ +Ġ×ŀ×IJ ×ķת +à¹Ħà¸Ľ à¹ģลà¹īว +Ġyapıl acak +ãĤ¹ãĤ¿ ãĥ¼ãĥĪ +ãģ» ãģ¼ +Ġko ÅŁ +ĠмаÑĤ еÑĢи +Ġsiè cle +ĠاÙĦÙħ ختÙĦÙģ +ĠاÙĦÙħختÙĦÙģ Ø© +Ġ׾ק ר×IJ +Ġ׾קר×IJ ת +Ġ×Ķפ ×ķ×¢×ľ +Ġt òa +Ġr Æ¡i +åij¨ ãĤĬ +à¸Ŀ à¸Ļ +j ÅĽÄĩ +ĠìķĬ ìĿĦ +اÙĨت ÙĤاÙĦ +ëĸ ł +ив аеÑĤ +ãĥĪ ãĥ« +ĠاÙĦÙģÙĦسطÙĬÙĨ ÙĬØ© +à¸ģลà¹Īาว วà¹Īา +ا Ùĥت +ĠÃĸ l +ĠÑĢе ÑĪи +ĠÑĢеÑĪи л +Ġ׳×ķס פ×ķת +Ġìłķ ì¹ĺ +вл еÑĩен +Ùħر ØŃÙĦØ© +Ġcome ça +Ġy ık +ìĤ ´ +à¸ĺ à¸Ļา +à¸ĺà¸Ļา à¸Ħาร +à¸Ńà¸Ļ า +à¸Ńà¸Ļา à¸Ħ +à¸Ńà¸Ļาà¸Ħ à¸ķ +Ġpeque ña +ä»ķ äºĭãĤĴ +Ġب Ø°ÙĦÙĥ +Ġнов ого +ãģĹãģ¦ ãģĦãģªãģĦ +ĠاÙĦÙħ ÙĬاÙĩ +à¸ģà¹ĩ à¹Ģà¸Ľà¹ĩà¸Ļ +Ġж ÑĥÑĢ +ĠжÑĥÑĢ нал +в еÑģ +خت ار +Ġ매 ìļ° +ĠM ã +ĠавÑĤомаÑĤ Ñĭ +ضع Ùģ +ĠاÙĦÙģ Ùĥر +ãģ§ãģĻ ãģ®ãģ§ +ãĥ¡ãĥ³ ãĥIJãĥ¼ +Ġк ÑĢÑĥг +ĠاÙĦسÙĦ طة +à¸Ħรัà¹īà¸ĩ à¹ģรà¸ģ +à¸ģระà¸Ĺ รว +à¸ģระà¸Ĺรว à¸ĩ +ÑĨ ов +éķ· ãģĦ +大ãģį ãģĦ +Ġgeç miÅŁ +ìĦ± ìĿ´ +Ġצר ×Ļ׼×Ķ +Ġм оÑī +ĠмоÑī н +Ġק ×Ļש +Ġק×Ļש ×ķר×Ļ×Ŀ +ĠNas ıl +г ÑĢан +Ġ×ŀ ×ķצר×Ļ×Ŀ +Ġ×ŀס ×ķ×Ĵ +Ġy ür +Ġyür üt +Ġ׾׊צ×ķ +×ķÖ ¼ +ĠìŀĪ ìĹĪëĭ¤ +Ġter ör +ĠTh Æ°Æ¡ng +ĠÙĪ ÙĬÙħ +ĠÙĪÙĬÙħ ÙĥÙĨ +ج ÙĪÙĨ +ĠÙĪغÙĬر Ùĩا +×ŀ פ×ķ +×Ĵ×ķר ×ŀ×Ļ×Ŀ +׼×ij ×Ļש +ĠاÙĦÙĦ غ +ĠاÙĦÙĦغ Ø© +شر Ùĥ +ĠاÙĦر اب +ĠاÙĦراب ع +ĠпÑĢ ек +ĠпÑĢек ÑĢаÑģ +ĠпÑĢекÑĢаÑģ н +Ġenerg ÃŃa +ק×ĵ ×ŀ×Ļ +ãģıãģª ãģ£ãģŁ +ĠÄij ứ +ĠÄijứ a +Serv i +Servi ço +Ġkald ır +åĥį ãģį +Ġод еж +Ġодеж д +물 ìĿĦ +ãģĿãģĨ ãģ§ +ãģĮãģĤ ãĤĮãģ° +ìĻ ķ +צ×ĵ ק +Ġart ır +Ġile ti +Ġileti ÅŁim +ãĤĪãģĨ ãģ§ +ãĥĪ ãĥ¼ +ãĤ¢ ãĥĭ +ãĤ¢ãĥĭ ãĥ¡ +×ĺ×Ļ ×Ļ׾ +ãĥķ ãĥªãĥ¼ +ãĥĿ ãĥ³ +ÐŁÑĢ о +Ġع اÙĦÙĬØ© +ĠÃ¶ÄŁ ret +ĠÃ¶ÄŁret men +ĠкаÑĩеÑģÑĤв а +Ġ×Ķ×ĺ ×ij×¢ +Ġзна Ñİ +ãģ¦ ãģıãĤĭ +Ġm ừng +ÙħÙĪ ت +ש ×ķ×ŀר +×Ĺ׾ ×ij +Ġwzgl ÄĻ +ĠwzglÄĻ du +ë²Ī 째 +Ġtá» ĵ +Ġtá»ĵ n +ãĥ¯ãĥ¼ ãĤ¯ +Ġpo życz +Ġpożycz k +×Ļ ×ķצר×Ļ×Ŀ +Ùĥر Ùħ +Ġг аÑĢ +ĠгаÑĢ ан +ĠгаÑĢан ÑĤи +ล à¹īาà¸ĩ +Ġìĺģ íĻĶ +×ĺ ×Ļס +Ġth ẻ +ĠìŀĪëĭ¤ ê³ł +اÙĦت ز +اÙĦتز اÙħ +Ġна ÑĪи +is ée +ãģĵãĤĮ ãĤĴ +Ġm ẽ +ض ÙĦ +بÙĪ ت +Ġ׼ ׼×Ķ +h ợ +ĠاÙĦس ÙĪرÙĬØ© +Ġ×ľ×¢ ×ķ×ŀ +Ġ×ľ×¢×ķ×ŀ ת +ĠbaÅŁ ar +ĠbaÅŁar ılı +е ÑģÑĤÑĮ +à¸Ħร ี +à¸Ħรี ม +ĠìłĦ ì²´ +ĠسÙĬ ÙĥÙĪÙĨ +Ġ×ŀ×ĵ ×ķ×¢ +ĠëķĮ문 ìĿ´ëĭ¤ +Ġc ứng +ger ät +Ġм иÑĢ +ĠмиÑĢ е +ĠÙĥÙĬÙģ ÙĬØ© +Ġפר ×ĺ×Ļ×Ŀ +Ġgo ÅĽci +иÑĤ еÑģÑĮ +ÑĥÑĪ ки +ؤ ÙħÙĨ +Ġ×IJ ׼ף +ĠاÙĦر جÙĦ +Ġl á»įc +à¹Ģรีย à¸ģวà¹Īา +ãģĵãģ® ãĤĪãģĨãģª +ë§Į íģ¼ +Ġп еÑĩ +ÙĪÙĦ ات +ĠÃľ ye +liÄŁ inde +à¸Ħะ à¹ģà¸Ļ +à¸Ħะà¹ģà¸Ļ à¸Ļ +ãĤĭãģĵãģ¨ ãģ¯ +วิ à¹Ģà¸Ħร +วิà¹Ģà¸Ħร าะ +วิà¹Ģà¸Ħราะ หà¹Į +Ġвозмож ноÑģÑĤи +ĠاÙĦÙĨ ساء +ãĥīãĥ© ãĥŀ +Ġgü c +Ġgüc ü +Ġt Æ°á»Ŀng +Ġacomp aña +ãĤ¤ ãĥ© +ק צ×ij +ĠY ö +ĠYö net +ĠYönet im +สัม à¸ľ +à¸ªà¸±à¸¡à¸ľ ัส +à¸Ļ าม +ĠÄij ợi +à¹ģหà¹Īà¸ĩ à¸Ĭาà¸ķิ +ãģĿãĤĮ ãģ§ãĤĤ +ät ig +ת ×ķ×Ŀ +ĠbaÅŁ lat +ĠвÑģ ей +ת ×Ļק +ת×Ļק ×ķף +ĠNg ô +ĠGesch ä +ĠGeschä fts +Ø£ Ùħ +Ø£Ùħ راض +à¹Ģà¸Ĺ à¸Ħà¸Ļ +à¹Ģà¸Ĺà¸Ħà¸Ļ ิ +à¹Ģà¸Ĺà¸Ħà¸Ļิ à¸Ħ +Ġм енÑĮ +ĠменÑĮ ÑĪе +Ġöl ç +Ġölç ü +ĠÙĬ جعÙĦ +ĠÄij ỡ +ש ×Ļ׾ +ש×Ļ׾ ×ķ×ij +ĠGr Ã¶ÃŁe +ĠÙĩ اتÙģ +รà¹īาà¸Ļ à¸Ńาหาร +×Ķ׾ ×Ļ׼ +×Ķ׾×Ļ׼ ×Ļ +иÑĢÑĥ ÑİÑī +èĭ¥ ãģĦ +ĠÃĸ zel +ãģĦãģŁ ãĤī +à¸Ħำ à¸ĸาม +Ġzosta ÅĤy +Ġ×Ķס ×Ļפ×ķר +×Ķ ×ķ׾ +×Ķ×ķ׾ ×ļ +à¹Ģà¸Ĭà¹Īà¸Ļ à¸ģัà¸Ļ +à¹Ĥ à¸Ĩ +à¹Ĥà¸Ĩ ษ +à¹Ĥà¸Ĩษ à¸ĵา +×IJר צ×ķת +×Ĵר פ×Ļ +Ġao ût +ĠÙĬ رÙĬد +ت ÙĪج +تÙĪج ÙĬÙĩ +ĠÑįÑĤ ап +ãĤ¹ãĤ¿ ãĥ³ +Ġkr ó +Ġkró tk +ãĤĴ使 ãģĨ +ì ·¨ +éĸ¢ ãĤı +à¸Ķà¹īวย à¸Ħวาม +à¸Ļำ à¹Ģสà¸Ļà¸Ń +Ġa yrıca +à¸Ī à¹īาà¸ĩ +ĠÑĦоÑĤ огÑĢаÑĦ +Ġв еÑĩ +ĠвеÑĩ еÑĢ +åĩº ãģĹãģŁ +ĠÐ¥ о +Ġ×ŀ ר×Ĵ×Ļש +à¹ĥหà¹ī à¹Ģà¸Ľà¹ĩà¸Ļ +ãĤĴ 缮 +ãĤĴ缮 æĮĩ +׾ ×ŀ×Ļ×Ŀ +nÄħ ÅĤ +ĠÑģÑĤ анд +ĠÑģÑĤанд аÑĢÑĤ +ĠSü d +ĠT âm +اخت بار +à¹Ģà¸ģ à¸Ńรà¹Į +Ùħس رØŃ +Ġbi á»ĩn +ب Ùı +Ġص اÙĦ +ĠصاÙĦ ØŃ +ĠPh ụ +íľ ´ +ãĥ¬ãĥĵ ãĥ¥ãĥ¼ +Ġbụ ng +Ġrég ime +ĠØ£ Ø´Ùĩر +ĠÑĢабоÑĤ ник +à¸Ŀ ัà¸Ļ +اع تÙħ +اعتÙħ اد +Ġзам еÑĤ +ãģ¾ ãģ£ãģ¦ +Ġch ặt +æĿ¥ ãĤĭ +ĠاÙĦÙĤ ÙĪات +ãģ«åħ¥ ãģ£ãģ¦ +تØŃ Ø§ÙĦÙģ +Ùħ زÙĬد +ĠÙĬ صÙĦ +ìĹ ¼ +à¹Ģà¸Ĭ à¹ĩ +à¹Ģà¸Ĭà¹ĩ à¸Ħ +Ġk á»ĭ +Ġká»ĭ p +ĠìķĦ ì§ģ +×IJ׳ ×Ĵ +Ġобла ÑģÑĤÑĮ +Ġpomoc Äħ +Ġ×ķ ש׾ +ëĵł ì§Ģ +ĠGi ám +ĠSt ück +Ġchá y +ĠëĤĺ ìĺ¤ +ש ×Ļ×ĺת +×ŀ×ĵ ר +×ŀ×ĵר ×Ļ×ļ +Ġsüre ç +к ва +×ij׾ ×Ļ×Ŀ +×Ķ ת×Ļ +×Ķת×Ļ ×Ļ×Ĺס +ÙĤب اÙĦ +Ġס ×ķ×Ĵ +Ġס×ķ×Ĵ ×Ļ +ÑģÑĤ олÑĮ +ä½ķ ãĤĤ +×ĸ׼ ×ķר +è²· ãģĨ +å®ī ãģı +à¸Ħรัà¹īà¸ĩ à¸Ļีà¹ī +kö p +ĠÑģеÑĢ виÑģ +оÑĩ нÑĭÑħ +ê±° ëŀĺ +تأ Ùĥ +تأÙĥ ÙĬد +×ĵ ׾ק +Ġпо Ñĩем +ĠпоÑĩем Ñĥ +пиÑģ аÑĤÑĮ +×ij שר +ĠH Ãłng +ĠT ìm +Ġtr ừ +ãĤ» ãĥĥãĤ¯ãĤ¹ +×ķ׳ ×Ĵ +mız da +п Ñģи +ĠìŀĪ 기 +Ġr út +ز اÙĨ +تÙĨ ÙĪع +ÙħÙĤ ا +ÙħÙĤا ÙĪÙħØ© +Ġ׾צ ×ķר×ļ +Ġ×ij ×Ļר×ķש׾×Ļ×Ŀ +ãĥ´ ãĤ£ +eb ile +ebile ceÄŁi +ãĥ¦ ãĥ¼ãĤ +ãĥ¦ãĥ¼ãĤ ¶ +ãĥ¦ãĥ¼ãĤ¶ ãĥ¼ +ãĤĴä½ľ ãĤĭ +Ñģ меÑĢ +ÑģмеÑĢ ÑĤ +Ġì§ ģ +Ġì§ģ ìłij +ĠÐŁ аÑĢ +ØŃ Ø§Ø¶ +ØŃاض ر +Ùħ ÙĥاÙģ +ÙħÙĥاÙģ ØŃØ© +ล ิà¸Ļ +ãģ¦ ãģįãģ¦ +ÑĢоÑģ л +ĠÄ°ÅŁ te +ÙĤص ÙĬر +Ġ×ij×Ĵ ×Ļ׾ +Ġ×ŀת ×IJ×Ļ×Ŀ +Ġ×Ķ ×Ĺ×ĵ +Ġ×Ķ×Ĺ×ĵ ש×Ķ +ר ×ķ×¢ +Ġprodukt ów +ĠÙħ صدر +не ÑĨ +ĠاÙĦعÙħÙĦ ات +Ġçık ma +Ġد بÙĬ +ק ×Ļף +ת ×IJר +ת×IJר ×Ļ×ļ +׳×Ļ ×Ļ×ĵ +صر اع +l ève +צ ×Ļר +à¸Ķ ัà¸Ļ +à¹ĥหà¹ī à¹Ħà¸Ķà¹ī +ãĤ¿ãĤ¤ ãĥł +Ġgi ảng +С ÐŁ +ĠاÙĦÙħ ØŃÙĦ +ĠاÙĦÙħØŃÙĦ ÙĬØ© +ĠT ất +׾ ×ķ×ĺ +h á»ķ +Ġam éric +Ġaméric ain +Ġ×ijש׾ ×ij +Ġ׾×IJ ×ķ×ŀ×Ļ +Ġpe ça +ĠÑĢаз нÑĭÑħ +ãģĦãĤĭ ãģ¨ +ãĥĩ ãĥ³ +ס קר +Ġ×Ķ×ŀ×Ĺ ×Ļר +ãģ¨ãģĦãģĨ ãĤĤãģ® +رت بط +ĠиÑģÑĤ оÑĩ +ĠиÑģÑĤоÑĩ ник +สมัà¸Ħร สมาà¸Ĭิà¸ģ +Ġ à¸Ĺัà¹īà¸ĩ +Ġà¸Ĺัà¹īà¸ĩ à¸Ļีà¹ī +ĠT áºŃp +ãģ£ãģ¦ ãģĦãģĨ +ĠاÙĦÙĪ صÙĪÙĦ +Ġdéc ada +Ġо ÑĦоÑĢм +ĠоÑĦоÑĢм лен +สำหรัà¸ļ à¸ģาร +Ġog óln +ãģĨãģ¡ ãģ« +Ġvá rias +ãģĻãģİ ãĤĭ +ÙĪ Ùĩا +à¹Ĥà¸Ľà¸£ à¸Ķ +ĠÐłÐ¾ÑģÑģ иÑı +人 ãĢħ +ãģĹãģ¦ ãģįãģŁ +Ġsı rasında +Ġng ôn +س ÙĨØ© +تÙħ تع +×ŀ׼ ×ij×Ļ +Ġnh ấn +×¢ ×ŀ×Ļ×ĵ +á» ¨ +ж иÑĤÑĮ +ãĤī ãģĽ +gr áf +gráf ica +ĠÙĤ ÙĪÙĦ +ĠÙĤÙĪÙĦ Ùĩ +ëĭ¨ ì²´ +ห à¹īา +หà¹īา ม +使 ãģ£ãģ¦ +ת ×Ļ×ij +ת×Ļ×ij ת +i á»ĥu +à¹ģ à¸Ĭม +à¹ģà¸Ĭม à¸Ľ +à¹ģà¸Ĭà¸¡à¸Ľ à¹Į +Ạ¬ +ĠëĤĺ ëĿ¼ +ĠÙħباشر Ø© +Ġtr Äĥm +سÙĥ ÙĪ +ĠاÙĦØ° Ùī +Ġbi ç +Ġbiç im +ت راجع +Ġоб еÑģп +ĠобеÑģп еÑĩ +ĠобеÑģпеÑĩ ива +Ġвозд ÑĥÑħ +Ñĭв аÑĤÑĮ +ÙĦ ØŃÙĤ +ĠMü dü +ĠMüdü rl +ĠMüdürl Ã¼ÄŁÃ¼ +Ġyapt ır +Ġפר ס +Ġפרס ×ķ×Ŀ +Ø· ÙĪر +ÑģÑĤв оваÑĤÑĮ +ìŀ¥ ìĿĦ +à¸Ĺีà¹Īà¸Ķี à¸Ĺีà¹Īสุà¸Ķ +à¸Ńั ล +ÑĢ Ñİ +Ùħست ÙĤبÙĦ +Ñģл ÑĥÑĪ +ÑģлÑĥÑĪ а +èªį ãĤģ +Ġ׾ ×Ļ×ŀ +Ġ׾×Ļ×ŀ ×ķ×ĵ×Ļ +ת ש×ķ×ij +תש×ķ×ij ×ķת +ĠgerçekleÅŁtir il +ĠاÙĦ اتÙģاÙĤ +ĠÑĥÑĢов не +ĠÑĤ ÑĢав +Ġ×Ķ×ŀ ×ķף +ØŃÙģ اظ +ĠÙħ ÙIJ +ĠÙħÙIJ ÙĨ +ĠÙħÙIJÙĨ ÙĴ +Ġdem ás +×ŀ×ķ×ĸ ×Ļק×Ķ +ש ×Ļ×Ĺ×Ķ +Ġb ú +алÑĮ нÑĭм +ãĤı ãģŁ +ãĤıãģŁ ãģĹ +ĠاÙĦÙħÙĪ اد +ת ׼׳ +×ª×Ľ×ł ×ķף +ãĥŃ ãĥĥãĤ¯ +hi ếu +ĠÑĥ ме +ÙħØŃا ÙĪÙĦØ© +×IJ ×ķשר +Ġкон кÑĥÑĢ +ĠконкÑĥÑĢ Ñģ +Ġ×ŀ ×ij×Ĺ +Ġ×ŀ×ij×Ĺ ×Ļ×ł×ª +Ġan lam +Ġanlam ı +Ġli á»ĩt +Ġв Ñħод +ĠH ình +ĠÙĨ ÙĬ +ĠÙĨÙĬ ÙĪز +ãĤ¸ãĥ£ ãĥ¼ +×ij ×Ļ×¥ +ÑĤелÑĮ нÑĭÑħ +à¸Ĺุà¸ģ à¸Ńยà¹Īาà¸ĩ +ĠkiÅŁ inin +Ø£ Ùĥثر +ĠиÑģÑĤоÑĢ ии +Ġë³Ģ íĻĶ +פ׾ ס×ĺ +×¤×ľ×¡×ĺ ×Ļ׳×Ļ +ĠÑģ еÑĤ +ĠÑģеÑĤ и +dıģ ımız +íķĺ ëıĦë¡Ŀ +×Ķ ר +×Ķר ×ij×Ķ +ãģĻãĤĭãģĵãģ¨ ãģ¯ +Ġphi ếu +تØŃ Ø³ÙĬÙĨ +ĠÅĽ rod +ĠÅĽrod ow +ĠÅĽrodow isk +ĠÑĢаÑģ Ñħод +بر ÙĬد +Ġر ÙĬ +ĠرÙĬ اÙĦ +Ġ×ķ ׼×ļ +ì§Ģ ìļĶ +׼ ×ŀ×ķ +Ġ×¢×ľ ×Ļ×Ķ×Ŀ +f ÃŃcio +Ġkar arı +tıģ ını +ĠС ов +ĠСов еÑĤ +ãģĬéĩij ãĤĴ +м еждÑĥ +междÑĥ на +междÑĥна ÑĢод +междÑĥнаÑĢод н +Ġm á»Ŀi +ĠاÙĦØ¥ ÙĬر +ĠاÙĦØ¥ÙĬر اÙĨÙĬ +ĠاÙĦرÙĪ سÙĬ +ص ÙĨد +صÙĨد ÙĪÙĤ +ĠاÙĦØ¥ÙĨ ترÙĨت +Ġt ắm +ĠÑĤак ого +Ġ×ij ׾×ķ×Ĵ +Ġü crets +Ġücrets iz +×Ĺ×ĸ ×Ļר +ìĸ´ ìķ¼ +ĠPh ần +ï¼ ľ +Ġ×ĺ ×ij×¢ +Ġ×ĺ×ij×¢ ×Ļ +×IJ×ŀ ×IJ +اÙĤ ÙĦ +Ġcondi ções +ÙĤات ÙĦ +ĠÑĢезÑĥлÑĮÑĤаÑĤ е +ĠÑģво ими +צ×ij ×Ļ×¢ +gé ni +Ġz es +Ġzes po +Ġzespo ÅĤ +ÑĪ ив +Ġפר×ĺ×Ļ ×ķת +Ùħست Ø´Ùģ +ÙħستشÙģ Ùī +شر ع +Ġko ÅĽci +Ġ×Ķ×IJ ×Ļ׳×ĺר׳×ĺ +ĠЧ еÑĢ +поÑĩ ÑĤ +Ġactiv ités +çŁ¥ ãģ£ãģ¦ +Ġ×ij ×ĸ×Ķ +Ġyüz den +ãģªãĤĬ ãģ¾ãģĽãĤĵ +Ġíĺ ¹ +Ġíĺ¹ ìĿĢ +Ġ×ŀש ׳×Ķ +ĠÐĴ еÑĢ +Ġ×ij×IJ×ķת ×ķ +éĿ¢ çĻ½ +éĿ¢çĻ½ ãģĦ +شر ØŃ +gr ünde +Ùģ Ø´ +ÙģØ´ ÙĦ +Ġsé jour +ë´ IJ +Ġr ôle +Ø´ عار +ем Ñĭе +ĠاÙĦج سÙħ +алÑĮ ное +Ġìĥģ íĥľ +ï¼ ¤ +ë¯Ģ ë¡ľ +ĠÙĨ ÙĤØ· +ĠÙĨÙĤØ· Ø© +ãģĿãģĨ ãģł +ãģĻãĤĭ ãģ®ãģĮ +ห ู +Ġnh á»ĭ +Ġeconóm ica +ס×ĺ ×ķ×ĵ +ס×ĺ×ķ×ĵ ׳×ĺ +มี à¹Ĥà¸Ńà¸ģาส +Ġgest ão +รูà¹ī วà¹Īา +Ġlo ạt +ĠاÙĦÙħ Ùı +ĠاÙĦØŃ ÙħÙĦ +ĠاÙĦعÙħÙĦ ÙĬØ© +Ġê²ĥ ëıĦ +ĠÐľÐ¾Ñģк ва +ק×ĺ ×ķר +Ġпод ÑĢоб +ĠподÑĢоб н +Ġl Æ°ng +ت Ùģس +تÙģس ÙĬر +ĠاÙĦ بع +ĠاÙĦبع ض +ئ ت +Ðķ ÐĿ +ìĹ° 구 +à¹ĥหà¹ī à¸Ħุà¸ĵ +ãģĤãĤĬ ãģ¾ãģĹãģŁ +Ġbir ka +Ġbirka ç +ĠÄ° sl +ĠÄ°sl am +çĹĽ ãģ¿ +Ġh ảo +Ġм аÑı +ĠiÅŁ çi +ש × +×©× ģ +à¸ģาร à¹Ģมืà¸Ńà¸ĩ +×ķ×Ķ ר +Ġch ó +ëĨ Ģ +Ġyan lı +Ġyanlı ÅŁ +幸 ãģĽ +×IJר×Ĵ ×ķ׳×Ļ +à¸Ńาà¸Ī าร +à¸Ńาà¸Īาร ยà¹Į +ĠинÑĦоÑĢм аÑĨиÑİ +Ðĵ Ðŀ +׳ ×Ĺש +ĠìķĮ ìķĦ +ĠÑħаÑĢакÑĤеÑĢ иÑģÑĤ +ĠÑħаÑĢакÑĤеÑĢиÑģÑĤ ик +à¸Ħุà¸ĵ สามารà¸ĸ +è¦ĭ ãģĪãĤĭ +à¸Ĭัà¸Ķ à¹Ģà¸Ī +à¸Ĭัà¸Ķà¹Ģà¸Ī à¸Ļ +ĠdziaÅĤ al +ĠdziaÅĤal noÅĽci +à¹Ĥà¸ŀ สà¸ķà¹Į +ĠÐļ ол +ĠÙģ ÙĩÙĬ +Ġ×ŀ פ׳×Ļ +Ġ×Ķק שר +Ùħر Ùĥ +ÙħرÙĥ ز +Ġho á +Ġа пп +Ġапп аÑĢаÑĤ +Ġp ami +Ġpami ÄĻ +ĠpamiÄĻ ta +Ġç ünkü +×ĵ ×ķף +ãģ¯ ãģĵãģ¡ãĤī +ĠM Ãł +ĠÙĬ ÙĤدÙħ +ĠпÑĢ ез +ĠпÑĢез иденÑĤ +à¸Ńุ à¸ķ +à¸Ńุà¸ķ สา +à¸Ńุà¸ķสา ห +à¸Ńุà¸ķสาห à¸ģรรม +ì§Ģ ìĽIJ +Ġ×IJפשר ×ķת +sch üt +schüt z +ĠTi ên +Ġsay ılı +ĠгÑĢÑĥпп Ñĭ +оÑĩ нÑĭй +Ġ×ľ×¢ ×ŀ×ķ×ĵ +Ġwr zeÅĽ +ĠwrzeÅĽ nia +ĠÄIJ ầu +à¹Ģà¸Ĥà¹īา รà¹Īวม +nız da +Ø®ÙĬ ص +Ġgü nc +Ġgünc el +ĠÙĦÙĩ Ø°Ùĩ +ĠÙĬ عتبر +lé gi +ãĤı ãģĭãĤĭ +Ġr ừng +ظ Ùĩ +ظÙĩ ÙĪر +Ġ×ŀ×ij ×Ļף +Ġ기 íĥĢ +åĪĩ ãĤĮ +lan mÄ±ÅŁ +à¸Ĺีà¹Ī มีà¸Ħวาม +Ġh á»ģ +ت ÙĪجÙĩ +ĠاÙĦØ¥ دارة +Ġú til +ס פ×ķ +à¸Ħวาม รัà¸ģ +à¹Ĥ ฮ +Ġпол иÑĤ +ĠполиÑĤ ик +Ġsat ın +ĠÅŀ imdi +×ŀ ×ķר×Ļ×Ŀ +ìķĺ ëĭ¤ +×Ĺ ×ķ×ķ +×Ĺ×ķ×ķ ×Ļ×Ķ +à¸Ħà¸Ńม à¸ŀิ +à¸Ħà¸Ńมà¸ŀิ ว +à¸Ħà¸Ńมà¸ŀิว à¹Ģà¸ķà¸Ńรà¹Į +Ġا ذا +تخ اذ +ãĤ¨ ãĥ« +Ġpossibilit é +ยืà¸Ļ ยัà¸Ļ +Ġü nivers +Ġünivers ite +ĠاÙĦد ÙĪرÙĬ +ĠìķĬëĬĶ ëĭ¤ +ĠìĦľ ë¡ľ +ØŃ Ø§ÙĦ +Ġë ¨ +Ġë¨ ¼ +Ġ먼 ìłĢ +à¸Ĺีà¹Ī à¸ĸูà¸ģ +ì§ ľ +Ġsk óry +лÑĮ ÑĨ +à¹ĥà¸Ĭà¹ī à¹Ģวลา +×ijק שת +ĠØ° ÙĪ +æĹ¥ ãĢħ +ĠкоÑĤоÑĢ ÑĥÑİ +ĠÑĥÑĢов енÑĮ +ê¹ ¨ +à¹Ħ à¸Ĺ +ãĤµ ãĥĹãĥª +ãĤ¸ ãĥ§ãĥ³ +ãģĻ ãģ¹ãģį +ĠG ór +ãĥĪ ãĤ¤ +ãĥĪãĤ¤ ãĥ¬ +ĠyaÅŁ ama +Ġdá»ĭ p +Ġb ữa +à¸ĭ ุ +Ġöl üm +ãģ£ãģ¦ ãģıãĤĭ +à¸ģาร à¸Ħà¹īา +ש ער +ĠÑĤип а +Ġг еÑĢ +ĠгеÑĢ о +רק ×¢ +Ġu waż +Ġuważ a +ש×ŀ ף +Ġhast alık +ãĤıãĤĮ ãĤĭ +ba ÅŁÄ± +Ñĩ ÑĤо +Ġ×ij ×ŀר׼×ĸ +Ġìļ°ë¦¬ ìĿĺ +ĠÙĥاÙĨ ÙĪا +ĠØ£ بر +Ġأبر ÙĬÙĦ +ì¸ µ +à¹Ħà¸Ĥ à¹Ī +ĠÙĪ ÙĦÙĪ +à¸Ĺ ัว +à¸Ĺัว รà¹Į +ĠÙĪØ£ Ùĥد +à¸Ĭ วà¸Ļ +׾ ×ķק +æį ¨ +æį¨ ãģ¦ +Ġİç in +p éri +Ġy al +Ġyal nız +ÑĮÑı н +Ġg ắng +à¸ģà¹ĩ ยัà¸ĩ +ĠУкÑĢа ин +ĠÑģ ами +ĠпÑĢовед ен +à¸ķà¸ģ à¹ģà¸ķà¹Īà¸ĩ +ĠQu ân +é paration +ĠbaÅŁ ında +Ġzn ale +Ġznale ź +Ġznaleź Äĩ +ãĤ± ãĥ¼ +ãĥİ ãĥ¼ +à¸ĸูà¸ģ à¸ķà¹īà¸Ńà¸ĩ +ëª ¸ +Ġëı Į +ĠëıĮ ìķĦ +ĠSch üler +Ġпод гоÑĤов +ĠподгоÑĤов к +ع رÙĪ +عرÙĪ ض +la ÅŁtır +ĠÑģоÑģÑĤав лÑıеÑĤ +ĠпÑĢоиз вод +ĠпÑĢоизвод ÑģÑĤва +ĠоÑģнов е +ĠØ´ ÙħاÙĦ +à¸ģร ี +ĠgörÃ¼ÅŁ me +оÑĩ ек +Ġ×Ĺ×ijר ×Ļ×Ŀ +ÙħØ® اط +Ùħخاط ر +ï¼ Ń +ר פ×IJ +ĠM ẹ +ยà¸Ńม รัà¸ļ +Ġv ết +Ø® Ø° +ĠاÙĦت Ø· +ĠاÙĦتط بÙĬÙĤ +à¸Ļ ึà¸ģ +Ġ×Ķ ×Ľ×ł×¡×ª +ĠогÑĢ ани +ĠогÑĢани Ñĩен +ĠÃĩ alÄ±ÅŁ +ĠاÙĦÙħÙĨت دÙī +à¸Īำà¸Ļวà¸Ļ มาà¸ģ +ĠÑĤоÑĢ ÑĢ +ĠÑĤоÑĢÑĢ енÑĤ +ĠìĤ´ ìķĦ +à¸ŀลัà¸ĩ à¸ĩาà¸Ļ +à¸Ĭ ัà¸Ļ +ĠÐIJн дÑĢ +Ġréalis é +×ŀש ×IJ +à¹ģ à¸Ĭ +à¹ģà¸Ĭ รà¹Į +Ġб ог +มา à¹ģลà¹īว +ĠاÙĦÙĨ ار +Ġolmad ıģı +×ĵ ×¢×Ķ +ĠÑĥ веÑĢ +ĠÑĥвеÑĢ ен +ãĤĭ ãĤĤãģ® +Ø£ د +أد ÙĪات +Ġ×Ķ×ĸ ×ķ×Ĵ +Ø¥ عÙĦاÙħ +h á»ı +ĠNä he +ĠÑĤ еÑģÑĤ +Ġ×ŀ ×ķ׼ר +Ġë¬¸ìłľ ê°Ģ +ת ×ķצ×IJ×Ķ +m ó +mó vel +ĠاÙĦتج ارة +Ġмног иÑħ +обÑī а +Ġ×¢ סק×Ļ +ĠEdu cação +ק ש×Ļ×Ŀ +é tabl +établ issement +Ġд еле +иÑĢÑĥ еÑĤÑģÑı +Ø¢ ثار +Ġ×Ķ×ŀ ר׼×ĸ×Ļ +ãĥIJ ãĥ« +ĠвÑģÑĤÑĢ еÑĩ +ãģĴ ãĤĭ +Ġci Äħ +ĠciÄħ gu +ÙĬ ست +à¸łà¸² ว +à¸łà¸²à¸§ ะ +Ø£ Ùħر +Ġо жи +Ġожи да +Ġ ủy +ãĥŀ ãĥ« +ر اس +оÑĩ ной +ת ×Ĵ×ķ×ij×ķת +تع رÙĬÙģ +ĠÑģо ÑĨиалÑĮно +ãĤĴ éĸĭ +ĠиÑģÑģлед ова +Ġd ú +Ġdú vida +Ġsk ÅĤ +ĠskÅĤ ada +Ġhä ufig +ĠвÑĭб ÑĢ +ĠвÑĭбÑĢ аÑĤÑĮ +ãģ®ãģ§ãģ¯ãģªãģĦ ãģĭ +ĠÑģ илÑĮно +ÑĤвеÑĢж ден +ר פ +רפ ×ķ×IJ×Ķ +æĢĿ ãģĦãģ¾ãģĻ +ØŃر ص +ש×ķת ×£ +Ùħس جد +à¹Ĥà¸Ĭ วà¹Į +ем ÑģÑı +в ÑĪие +Ġм л +Ġмл н +Ġ׾×Ķ ×ij×Ļ×IJ +ĠÙĬ تعÙĦÙĤ +à¸ķ ูà¹ī +Ġп ÑĢаз +ĠпÑĢаз д +ĠпÑĢазд ник +Ġн ем +Ġнем ного +Ġs Ãłng +تÙĨ سÙĬ +تÙĨسÙĬ ÙĤ +Ġtá» Ŀ +Ġмед и +ãģ« æĪ +ãģ«æĪ » +à¸Ħว à¹īา +ãģĭ ãģijãĤĭ +×ij׾ ×ķת +ĠÑįк Ñģп +ĠÑįкÑģп еÑĢÑĤ +Ġдев ÑĥÑĪ +ĠдевÑĥÑĪ к +ĠØŃ Øµ +ÙĨØ´ Ø£ +ãģĮãģĤãĤĭ ãģ®ãģ§ +Ġت راÙħ +ĠتراÙħ ب +أس ÙĪاÙĤ +Ġ׾פ ׳×ķת +Ġا ï»· +ãģ« ãģı +ãģ«ãģı ãģĦ +ĠØ£ عÙĦÙī +Ġ׾×Ķ ×ŀש×Ļ×ļ +rä u +ש×ŀ ×Ļ×Ŀ +åĪĨ ãģij +ãģĻ ãģ§ +ãģĻãģ§ ãģ« +×Ķ׾ ׼×Ķ +×Ĺ׾ ×Ļ×£ +Ġì ±ħ +Ġì±ħ ìŀĦ +à¹Ģà¸Ī ริ +à¹Ģà¸Īริ à¸į +éģĬ ãģ³ +ج سد +สา à¸ĺ +สาà¸ĺ าร +สาà¸ĺาร à¸ĵ +Ġbas ın +ÑĢаР³ +г ад +Ġho ÅŁ +íķ µ +×ij×Ĺ ×Ļר×Ķ +×ŀס ×ļ +Ġìłľ íĴĪ +تÙħ ÙĪÙĬÙĦ +ĠL Æ°u +ë¡ľ ë¶ĢíĦ° +Ġп об +Ġпоб ед +ÙħÙĨ Ø° +常 ãģ« +ÙĤ س +ĠاÙĦÙħ صدر +ĠÙĪاÙĦ است +Ġkh ắp +ĠاÙĦج اÙĨب +Ġng uyá»ĩn +éĸĵ éģķãģĦ +ĠÑģÑĤ ÑĢа +ĠÑģÑĤÑĢа Ñħ +ĠÑģÑĤÑĢаÑħ ов +รี à¸ļ +Ġx Æ°Æ¡ng +Ġì° ¾ +Ġì°¾ ìķĦ +Ġng ại +г ал +à¸ĭ ีà¹Ī +Ġ×ij פ×Ļ×Ļס×ij×ķק +Ц енÑĤÑĢ +Ġaval iação +Ġeconóm ico +×ĸ ף +ĠÐľ ак +Ġinter és +à¸ģล ิà¹Īà¸Ļ +ÑģÑĤÑĮ Ñİ +ĠÄij Æ°Æ¡ng +å¼· ãģı +ĠKh ách +à¹Ģà¸Ļืà¹īà¸Ń หา +ĠYaz ı +è²· ãģ£ãģ¦ +Ðł Ðķ +à¹Ģà¸ŀิà¹Īม à¸Ĥึà¹īà¸Ļ +สม à¸ļู +สมà¸ļู รà¸ĵà¹Į +Ġм иÑĢов +×Ĵ ׳×Ļ×Ŀ +ĠÄij ức +à¸Ń ารà¹Į +ص اص +ãģĬ ãĤĪ +ãģĬãĤĪ ãģ³ +ÃªÌ ī +ĠاÙĦÙħؤ تÙħر +ĠاÙĦÙħر ØŃÙĦØ© +สà¸Ńà¸ļ à¸ĸาม +Ġà¸Īาà¸ģ à¸Ļัà¹īà¸Ļ +Ġت عد +ãģĿãģ® ãģŁãĤģ +Ġkh áng +à¸Ļ ิà¸Ķ +ãĥĬ ãĥ³ +ëĦ¤ ìļĶ +ĠاÙĦ اØŃت +ĠاÙĦاØŃت ÙĦاÙĦ +ìļ ķ +Ġмод ели +ĠпÑĢоÑĨ енÑĤ +à¸ŀวà¸ģ à¹Ģรา +Ġ×Ķצ ×ĵ +Ġ×Ķצ×ĵ ×ĵ×Ļ×Ŀ +ständ e +׳ ×Ĵר +Ġdot yc +Ġdotyc zÄħ +ĠdotyczÄħ ce +ĠÅĽ wiÄĻt +×ŀר ×Ķ +ãģĻãģĶ ãģĦ +ãĥĩãĤ£ ãĥ³ãĤ° +à¸ģาร สรà¹īาà¸ĩ +ë Ĥ¬ +Ġì°¸ ìŬ +Ñģ Ñħ +ÑģÑħ ем +ÙħÙĪ س +Ġn ấu +Ġ׾×ŀ×¢ ׾×Ķ +à¹Ģà¸Ľ à¹īา +à¹Ģà¸Ľà¹īา หมาย +Ġmù i +ائ ز +íĽ Ī +×Ĺ×ij ×ķר×Ķ +à¸ľà¸¹à¹ī à¹ĥà¸Ĭà¹ī +Ġpa ź +Ġpaź dzi +Ġpaździ ern +Ġpaździern ika +ลà¸ĩ à¹Ħà¸Ľ +ÙĤ اع +Ġch áºŃm +Ġözellik leri +ĠÄIJ o +ĠÄIJo Ãłn +ж ение +Ġh ẳ +Ġhẳ n +ĠaÅŁ k +ï½ į +ãĥij ãĤ¹ +×Ķ×ķר ×IJ×ķת +ĠÅ » +ĠÅ» y +×ŀ×ĸ ׾ +ĠÑĥ кÑĢа +ĠÑĥкÑĢа ин +à¹Ģà¸Ĭ ิ +à¹Ģà¸Ĭิ à¸į +Ðł Ðĺ +ĠzwiÄħz ku +×Ķ×Ĺ׾×ĺ ת +ãĤĵãģ§ãģĻ ãĤĪãģŃ +ãģ¦ ãģĬãĤĬ +лож иÑĤÑĮ +×ŀ ×ķ׳×Ļ×Ŀ +ฮ ิ +ì° ¬ +ĠاÙĦÙħØ´ ترÙĥ +ĠdÃ¼ÅŁ ük +аг енÑĤ +ĠاÙĦØ£ سبÙĪع +ĠÙĤ رÙĬب +ин д +инд ив +индив ид +индивид Ñĥ +индивидÑĥ алÑĮн +för der +Ġseç en +Ġseçen ek +Ġét ant +ĠлÑİб им +каз ÑĭваеÑĤ +ว ิà¸Ļ +Ġ×Ķ×ij ×IJ×Ļ×Ŀ +Ġд ов +Ġдов олÑĮ +ĠдоволÑĮ но +×¢×ĵ ×Ļ×£ +Ġok re +Ġokre ÅĽ +ĠokreÅĽ lon +Ġت رÙĬد +à¹Ģมืà¹Īà¸Ń วัà¸Ļà¸Ĺีà¹Ī +ãĤĪ ãģĭãģ£ãģŁ +Cum h +Cumh ur +Cumhur ba +Cumhurba ÅŁ +CumhurbaÅŁ kan +CumhurbaÅŁkan ı +Ġn ợ +à¸ľà¸¹à¹ī à¹Ģลà¹Īà¸Ļ +Ġcompl ète +à¹Ģà¸ŀ ศ +د ÙIJ +Ġdü z +Ġdüz ey +ãģ§ãģĤãĤĭ ãģĵãģ¨ +ext érieur +× ³ +Ġinform ação +ãĤ¯ãĥª ãĥĭãĥĥãĤ¯ +ĠPub li +ĠPubli é +ר ×ķ×ĵ +à¸Ħวาม à¸Ľà¸¥à¸Ńà¸Ķà¸łà¸±à¸¢ +ĠØ£ÙĬ ض +ĠØ£ÙĬض Ùĭا +ت سبب +ãģ¤ ãĤĤãĤĬ +из ма +à¸Ĥึà¹īà¸Ļ à¹Ħà¸Ľ +Ùĥ ÙIJ +ÙĦ ÙĪÙħ +Ġש צר +Ġשצר ×Ļ×ļ +ãģ¯ ãĤĤãģ¡ãĤįãĤĵ +Ġк ан +Ġкан ал +ãģ«ãģª ãģ£ãģ¦ãģĦãģ¾ãģĻ +ĠاÙĦØ£ Ùĥثر +ت اØŃ +ÙĨت Ùĩ +ÙĨتÙĩ اء +ا ÙĪÙĬØ© +ĠBug ün +н Ñģкого +à¸Ķ à¹Īวà¸Ļ +é volution +ãģ£ãģ¦ ãģĦãģ¾ãģĹãģŁ +ãĤ ħ +ĠV Æ°Æ¡ng +à¸łà¸²à¸ŀ ย +à¸łà¸²à¸ŀย à¸Ļ +à¸łà¸²à¸ŀยà¸Ļ à¸ķรà¹Į +Ġ×Ķ צ׾×Ļ×Ĺ +ĠاÙĦإسÙĦاÙħ ÙĬ +ÙĦÙĬ ب +Ġed ição +ÑģÑĤÑĢ ел +Ġkh úc +ÙĨÙħÙĪ Ø° +ÙĨÙħÙĪØ° ج +׾ צ×Ķ +ÑģÑĤав ил +à¸ĸ า +สรà¹īาà¸ĩ à¸Ħวาม +ãģĦ ãģ£ãģ± +ãģĦãģ£ãģ± ãģĦ +ÑģÑĤав лен +ĠاÙĦ ÙĤدس +Ġng ược +ب Ø® +ส หร +สหร ั +สหรั à¸IJ +ĠØ£ غ +Ġأغ سط +Ġأغسط س +ãģĨ ãģ¾ +ãģĨãģ¾ ãģı +ĠêµŃ ìłľ +ØŃض ار +Ġd ừng +æĬ¼ ãģĹ +ت ÙĪا +تÙĪا جد +ש×ŀ ×Ĺ×Ķ +ãģı ãĤĵ +Ġ×ij×¢ צ +Ġ×ijעצ ×Ŀ +×ŀ ׳×Ļ×ķת +×ķ ×Ļ×ĵ +×ķ×Ļ×ĵ ×IJ×ķ +à¸Ĭ ิà¸ĩ +Ġprac ÄĻ +Ġз аÑĤ +ĠзаÑĤ ем +ĠìŀIJ ìľł +Ġì¤ Ģ +Ġì¤Ģ ë¹Ħ +Ġb áºŃ +ĠbáºŃ c +Ġ×Ķ×ŀ צ×ij +ĠÙĤ ÙĬÙħØ© +à¹Ģà¸Ń à¹Ģà¸Ĭ +à¹Ģà¸Ńà¹Ģà¸Ĭ ีย +Ġperch è +ĠاÙĦع سÙĥر +ĠاÙĦعسÙĥر ÙĬØ© +ج ÙĬب +ëŀ µ +Ùħ Ùĩر +ÙħÙĩر جاÙĨ +Ùħ راÙĥ +ÙħراÙĥ ز +Ġод нако +à¸Ķี à¹Ĩ +Ġצ פ×ķ +Ġkullan ılan +Ġк ино +ãĥĨãĤ£ ãĥ³ãĤ° +ĠGi Ỽi +ت ÙĪز +تÙĪز ÙĬع +ย ิà¸Ļ +ยิà¸Ļ à¸Ķี +Ġc Åĵur +ĠiÅŁ aret +Ġ×ij×¢ ×ĸר +Ġ×ij×¢×ĸר ת +Ġп аÑĨи +ĠпаÑĨи енÑĤ +ãģ¿ãģŁãģĦ ãģ§ãģĻ +в ез +ли на +од е +Ġ×IJ×ķת ף +dıģ ınız +ĠÐIJ в +ĠÐIJв ÑĤоÑĢ +ï¼ ® +ĠC ần +ĠاÙĦا Ø® +ĠاÙĦاخ بار +Ġê±° ìĿĺ +Ġat enção +Ġgeld iÄŁi +ãĤª ãĤ¹ +ãĤªãĤ¹ ãĤ¹ +ãĤªãĤ¹ãĤ¹ ãĥ¡ +ев Ñĭе +кÑĢÑĭ л +à¹Ģà¸Ĭ ียà¸ĩ +à¹Ģà¸Ĭียà¸ĩ à¹ĥหมà¹Ī +Ġmar ço +ĠاÙĦÙħ ادة +Ġг ол +Ġsprzeda ży +Ġíķ´ ê²° +ĠÐķ го +ê¹ Ģ +Ġ׾ק×ij׾ ת +ĠاÙĦÙģ ÙĨاÙĨ +Ġcomunic ación +à¹Ģสà¹īà¸Ļ à¸Ĺาà¸ĩ +íĺ ¹ +à¸Ĭ ำ +à¸Ĭำ ระ +Ġ׼ ×IJ×ŀ +Ġ׼×IJ×ŀ ×ķר +à¸Ĭ à¹Īาà¸ĩ +ز Ùĩر +Ġklient ów +ива ÑİÑĤ +ан г +׳ ×ļ +Ġg á»įn +Ãľ R +ìĺģ ìĥģ +Ġغ زة +ìĿĮ ìĿĦ +Ġbez po +Ġbezpo ÅĽ +ĠbezpoÅĽ redni +ĠاÙĦÙħ ÙĪا +ĠاÙĦÙħÙĪا Ø·ÙĨ +ĠاÙĦÙħÙĪاطÙĨ ÙĬÙĨ +ãĤĮ ãģ¾ãģĻ +ĠмаÑĤ Ñĩ +×IJ ×ķף +Ġر سÙħÙĬ +ĠÑįк он +ĠÑįкон ом +ĠÑįконом иÑĩеÑģк +ãĥľ ãĥ¼ +Ġд иÑĢ +ĠдиÑĢ екÑĤоÑĢ +ĠÑģк оÑĢо +à¸ļ ำ +à¸ļำ ร +à¸ļำร ุà¸ĩ +ĠÑĦ ÑĥÑĤ +ĠÑĦÑĥÑĤ бол +Ġ×IJ ×Ļ׾ +Ġì¤ij êµŃ +ìľ ¤ +eÄŁ e +à¹Ħ à¸ģà¹Ī +tra î +traî n +ĠÑĤ ÑĢÑĥб +à¹Ģà¸ļ ื +à¹Ģà¸ļื à¹īà¸Ńà¸ĩ +à¹ģม à¸Ļ +ĠتØŃ Ø¯ÙĬØ« +Ġ׼ עת +ØŃ Ø§Ø³Ø¨ +lı ÄŁa +ק×Ļ ×Ļ×ŀ×Ļ×Ŀ +оÑģÑĤ ÑĮÑİ +à¸Ŀ ั +à¸Ŀั à¹Īà¸ĩ +Ø´ غÙĦ +ìĽ ¹ +Ġкажд ого +Ġbölüm ü +หà¸Ļ ี +Ġistedi ÄŁi +Ġtr Æ°ng +ãĥ Į +ฮ à¸Ń +Ø£ÙĨ Ø´ +Ø£ÙĨØ´ طة +ĠاÙĦÙħ سÙĬ +ĠاÙĦÙħسÙĬ ØŃ +ลัà¸ģษ à¸ĵà¹Į +Ġn á»Ńa +à¸Ĺีà¹Ī à¸ķà¹īà¸Ńà¸ĩà¸ģาร +ÑĪ ек +л Ñij +Ġש ×Ļ×Ķ +Ġש×Ļ×Ķ ×Ļ×Ķ +Ġkhu ôn +ĠÑĤÑĢеб ованиÑı +Ġ×ľ×¢ ×ĸ×ķר +ĠاÙĦع Ùħر +ราà¸Ħา à¸ĸูà¸ģ +ÙĩÙı ÙħÙĴ +ü st +üst ü +Ġден ег +Ġn ạ +à¸Ĥà¸Ļ ม +Ġбл аг +Ġблаг од +Ġблагод аÑĢ +ĠблагодаÑĢ Ñı +Ø¥ سÙĦاÙħ +à¸Ļิ ว +çŁ¥ ãĤīãģªãģĦ +Ø« ÙĤØ© +Ġг олоÑģ +×IJ×ķר ×Ĺ +Ġtr ứng +Ġод ном +ĠkoÅĦ cu +Ġ×ķ רק +Wi ÄĻ +WiÄĻ cej +Ġ×IJ ×Ļ׼×ķת +Ġ×IJ×Ļ׼×ķת ×Ļ +Ñģ оÑģ +Ġje żeli +以ä¸ĭ ãģ® +å°ı ãģķ +å°ıãģķ ãģª +олог ии +Ġоб ÑģлÑĥж +ĠобÑģлÑĥж ива +Ùĥت ابة +Ġê´Ģ ìĭ¬ +×¢ ש×Ļר +Ġaras ındaki +ĠÑĢай она +ÙĪا جب +Ġ×ij×Ĺ×Ļ ×Ļ +íķ´ 주 +Ġg óc +ай л +ĠT ình +æļ® ãĤī +æļ®ãĤī ãģĹ +æĻĤ ãģ«ãģ¯ +ĠгоÑĢод е +Ġ׼×IJ ×Ļ׾ +Ġ׼×IJ×Ļ׾ ×ķ +ĠC á»Ļng +ãģ©ãģĨ ãģĹãģ¦ãĤĤ +×Ĺ ×ķ×£ +تØŃ Ø±Ùĥ +ĠÑģлов ам +à¸Īะ à¸Ĭà¹Īวย +ĠاÙĦÙħست ÙĤبÙĦ +ÙĤ ض +ÙĤض ÙĬ +×ijס ×ķפ +×ijס×ķפ ×ķ +iÄĻ Äĩ +ĠY ıl +Ø´ ÙĬØ® +à¸Ħุà¸ĵ à¸Īะ +ש×ŀ ×ķת +Ġت عرض +Ġanál ise +ĠÑģоб иÑĢа +à¹Ģà¸ŀ à¸Ĭ +à¹Ģà¸ŀà¸Ĭ ร +Ġв ели +Ġвели к +สั à¹īà¸Ļ +Ġpop ulação +รà¹Īวม à¸ģัà¸Ļ +×Ĺ ×ŀ +×Ĺ×ŀ ×Ļש×Ļ +ס ×Ļס +åĨħ ãģ§ +Ġsob Äħ +ĠY ay +ĠYay ın +ãĥ¡ ãĥĭãĥ¥ãĥ¼ +ĠпÑĢедоÑģÑĤав лÑı +ãģł ãģ¨æĢĿãģĨ +Ġê³ł ê°Ŀ +Ġод ним +à¹ĥà¸Ļ à¹Ģรืà¹Īà¸Ńà¸ĩ +Ġs á»ķ +ĠÐĹ Ð´ÐµÑģÑĮ +Ġизмен ениÑı +ĠìĿ¼ ìĿĦ +ãģªãģ® ãģł +клад Ñĭва +ÑĢ ма +Ġ×ķ×ij ׼׾ +تأ ÙħÙĬÙĨ +ĠпÑĢи ÑıÑĤ +ĠпÑĢиÑıÑĤ н +Ùħ Ùħار +ÙħÙħار سة +ãģ¨ãģª ãģ£ãģ¦ +Ġج ÙħÙĬÙĦ +Ġì§ Ī +Ġì§Ī 문 +Ġquest ão +i é +ié ndo +หà¹īà¸Ńà¸ĩ à¸ŀัà¸ģ +ãĥij ãĥ¼ãĥĪ +ÑĤвеÑĢж да +н Ñģкой +з ал +มุ à¹Īà¸ĩ +á» Ĭ +Ġ×Ķ×IJ×Ĺר ×ķ׳×Ķ +ĠTh Æ° +주 민 +ĠاÙĦع ب +év én +évén ement +ÙĤÙĪ اعد +د Ùı +ĠìķĬ ìĬµëĭĪëĭ¤ +Ġë³´ 기 +Ġyapıl ması +à¹Ģร าà¸ģ +à¹Ģราà¸ģ à¹ĩ +ØŃ Ø°Ø± +ÙĤ صر +ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĹãģŁ +Ġà¹Ģà¸Ľà¹ĩà¸Ļ à¸ķà¹īà¸Ļ +ãģ¨ ãģ« +ãģ¨ãģ« ãģĭ +ãģ¨ãģ«ãģĭ ãģı +н ÑĨе +зв Ñĥк +ãģĹãĤĪãģĨ ãģ¨ +ĠاÙĦصØŃ ÙĬØ© +Ġש×Ķ ×Ļ×ķ +ĠDi ÄŁer +ÙĤÙĦ ÙĤ +ãĤ¸ãĥ£ ãĥ³ +Ġr á»Ŀi +Ġл еÑĩ +ĠлеÑĩ ениÑı +تب اد +تباد ÙĦ +צ פ×Ķ +à¸Ħวาม à¹Ģหà¹ĩà¸Ļ +ĠØ´ ب +Ġشب ÙĥØ© +ר ×Ļק +Ùħ عد +Ùħعد ات +dıģ ında +Ġ×ijש ׳×Ļ×Ŀ +Ġ×Ķ ×Ļשר×IJ׾ +Ġ×Ķ×Ļשר×IJ׾ ×Ļת +Ġsı nav +׳צ ×Ļ×Ĵ +วัà¸ķ à¸ĸุ +ĠاÙĦبر ÙĦÙħ +ĠاÙĦبرÙĦÙħ اÙĨ +t ivitÃł +ãĤĵãģł ãĤįãģĨ +ק×Ļ ×Ļ×ŀ +ÙĦÙĬ Ùĥ +ĠÄij ò +ĠÄijò i +ĠÐĺн ÑĤеÑĢ +ĠÐĺнÑĤеÑĢ неÑĤ +ãģ«ãģ¨ãģ£ãģ¦ ãģ¯ +ãģ£ ãģĵ +ק ×ķס +ست ØŃÙĤ +æķĻ ãģĪãģ¦ +ãĥĢ ãĥ¡ +ĠÙħÙĨ زÙĦ +à¹Ģà¸ĭ à¹ĩà¸Ļ +使 ãģĪãĤĭ +è¦ĭ ç©į +è¦ĭç©į ãĤĤãĤĬ +Ø£ Ùģ +Ø£Ùģ Ùĥار +Ġиг ÑĢов +ĠигÑĢов Ñĭе +Ġm ÄĻż +ĠmÄĻż czy +ĠmÄĻżczy zn +ĠاÙĦØŃ ÙĤÙĬÙĤÙĬ +ع بر +׼×ķ׾ ׳×ķ +íĿ ¥ +×ŀ×IJ ×ķ×Ĺר +خت ص +ãĥŀ ãĥŀ +Ġ×IJ×Ĺ ×ķ×ĸ +í ĮĢ +Ġr á»iji +Ġв ÑĤоÑĢ +ĠвÑĤоÑĢ ой +Ġl ẫn +пÑĢ ом +пÑĢом ÑĭÑĪ +пÑĢомÑĭÑĪ лен +пÑĢомÑĭÑĪлен н +ĠоÑĤноÑĪ ениÑı +Ġs ứ +Ġм обилÑĮ +ĠмобилÑĮ н +ĠÑįÑĤ омÑĥ +Ġt ạp +ĠìĤ¬ ê±´ +ĠìķĮ 볤 +Ùĥ Ùı +ÙĥÙı ÙħÙĴ +Ġק ×ķר×Ķ +ĠÑĦ иÑĢ +ĠÑĦиÑĢ м +Ġsık ıntı +׳ ׼ +׳׼ ×ķף +ÙĪÙĦÙĪج ÙĬ +ØŃ Ø§ÙĨ +Ġlo ạn +Ġ×IJ׾ ×£ +Ġm ắn +abh äng +abhäng ig +ĠÑĥÑĢов нÑı +Ġ׾×ij×ĵ ×ķק +ÙĬ ÙħÙĨ +lay ın +Ġh ải +Ġзав од +ĠìķĦ 주 +สà¸ĸ า +สà¸ĸา à¸ļัà¸Ļ +Ġgüven lik +à¹Ģà¸Ķ à¹Īà¸Ļ +×ij×ĵ ק +Ġë Ī +ĠëĪ Ħ +ĠëĪĦ 구 +éĩįè¦ģ ãģª +รà¸Ńà¸ĩ รัà¸ļ +sch lie +schlie ÃŁen +Ġìĸ ¼ +Ġìĸ¼ ë§Ī +Ġìĸ¼ë§Ī ëĤĺ +ÑĤи ки +íķľëĭ¤ ê³ł +ãģłãģ£ãģŁ ãĤī +Ġ×Ķ ×Ļ×ĺ×ij +ãģªãģijãĤĮãģ° ãģªãĤīãģªãģĦ +â Ì +Ã¢Ì £ +Ġph ạt +ak Ä±ÅŁ +ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĻ +à¹Ģà¸ĭ à¹ĩ +ĠС егоднÑı +Ġinsan ların +Ġdévelop pe +ת פר +תפר ×Ļ×ĺ +اÙĨت شار +ê° ij +Fran çois +Ø£ÙĦ ع +Ø£ÙĦع اب +ãĤĴ è¶ħ +ãĤĴè¶ħ ãģĪ +Ġê°Ļ ìĬµëĭĪëĭ¤ +ãĤ³ ãĥ¬ +ĠмеÑģÑı ÑĨев +íĮ ħ +ĠاÙĦج اÙħعة +ìĿ¸ íĦ° +ìĿ¸íĦ° ëĦ· +×ĵר ×ķש +ĠÙĪØ£ شار +ĠпÑĢав ила +ãģĿãģĵ ãģ« +×Ĺ ×ŀ×ĵ +à¹Ģหà¸ķุ à¸ģารà¸ĵà¹Į +Ġê²½ íĹĺ +ãģ¶ ãĤĬ +׾ ש +׾ש ×ķף +à¹Ģ à¸ĸ +ĠDo ÄŁu +ĠиÑģполÑĮзов ание +Ġçoc uÄŁu +магазин е +ĠÄiji á»ĥn +Ġas lı +Ġaslı nda +Ġdoen ça +Ġس اع +Ġساع ات +ĠиÑģполÑĮзов аниÑı +ר ×ķצ×Ļ×Ŀ +ĠзнаÑĩ иÑĤ +ĠÑĢаР¼ +ĠÑĢам каÑħ +ê±° 리 +Ġп ÑĭÑĤа +ãĥģ ãĥ³ +Ġпо Ñģк +ĠпоÑģк олÑĮ +ĠпоÑģколÑĮ кÑĥ +Ø¥ بر +إبر اÙĩ +إبراÙĩ ÙĬÙħ +ĠÑĤÑĢ еÑħ +ĠGen ç +س ÙĪÙģ +Ġve ÃŃculo +ĠNg ân +ĠоÑĩеÑĢ едÑĮ +à¸Ħร ึà¹Īà¸ĩ +×IJ ×ij×Ļ +à¸ķ à¹īม +ãĤĴè¡Į ãģĦ +ĠاÙĦساب ÙĤØ© +на ÑĨи +наÑĨи она +наÑĨиона лÑĮн +Ġgest ión +ت ÙĤد +ĠاÙĦبÙĬ اÙĨ +ĠاÙĦبÙĬاÙĨ ات +ĠاÙĦ اÙĨتخاب +ĠاÙĦاÙĨتخاب ات +à¹Ģà¸Ĭ à¹Īา +×ĵ ×IJ×Ĵ +Ġ׾×Ĵ ×ŀר×Ļ +Ġت ØŃتاج +Ġth ôn +à¸ķ à¹īà¸Ńà¸Ļ +à¸ķà¹īà¸Ńà¸Ļ รัà¸ļ +女 ãģ® +女ãģ® åŃIJ +Ġth ợ +Ø· ØŃÙĨ +ารà¹Į à¸Ķ +ת ×ŀ×Ļ×ĵ +ĠÑģам Ñĭм +Ġìĭľ íĸī +Ø¥ صد +إصد ار +ĠNgh á»ĩ +ìķ ķ +س ئ +سئ ÙĦ +à¸Ń าร +à¸Ńาร ม +à¸Ńารม à¸ĵà¹Į +à¹ģ ฮ +׳×ĺ ׾ +Ġì¢ĭ ìķĦ +×ķ׾ ׾ +Ġ×ij ×Ľ×ª×ij +ãĤ« ãĥ© +צע ×Ļר×Ļ×Ŀ +تعب ÙĬر +Ġ×ŀ קר×Ķ +ĠÑĦак ÑĤоÑĢ +Ġت ÙħاÙħ +ĠتÙħاÙħ ا +ëį ķ +Ġv Æ°á»Ŀ +ĠvÆ°á»Ŀ n +Ġd Ä±ÅŁÄ± +ãģĦ ãģ¡ +Ġ׾ק ׳×ķת +ĠاÙĦع ÙĦاÙĤات +п Ñĥб +пÑĥб ли +Ø¥ ÙĬÙħ +Ø¥ÙĬÙħ اÙĨ +à¸Ńำ à¸Ļา +à¸Ńำà¸Ļา à¸Ī +åIJ« ãģ¾ãĤĮ +ãĤĭ ãģŁãĤģãģ« +ס ×Ĵ +ס×Ĵ ׳×ķף +تØŃ Ø¯ÙĬ +Ġaup rès +ĠاÙĦج Ùĩا +ĠاÙĦجÙĩا ز +Ġ×ŀ ת×Ĺת +ен нÑĥÑİ +Ġз им +à¸ģา à¹ģà¸Ł +Ġ×ijת ×ķר +Ġngh è +Ġnghè o +ĠÐĽ Ñİ +ĠÐĽÑİ Ð± +תק צ×Ļ×ij +×ŀ×¢ ש×Ķ +ĠاÙĦبÙĬ ت +צ ×Ļפ +ĠобÑıз ан +ĠM á»Ĺi +ĠТ ÑĥÑĢ +ĠÙĪب اÙĦت +ĠÙĪباÙĦت اÙĦÙĬ +Ġdéc ision +Ġب د +Ġبد أت +Ġc ục +Ġb ask +Ġbask ı +Ġhat ırl +Ġhatırl a +å°ı ãģķãģĦ +Ġgerçek ten +à¸ľ ัà¸ģ +åı¯èĥ½ ãģª +×ŀ×IJ ס +Ġcr ÃŃtica +ĠìĿĺ ìĽIJ +عÙĤ ÙĪد +×ĺ ׼׳ +×ĺ׼׳ ×ķ׾×ķ×Ĵ×Ļ×Ķ +è¨Ģ ãģĪãģ° +ĠÙĤ ÙĨا +ĠÙĤÙĨا Ø© +ĠìĿ´ê²ĥ ìĿĢ +ت صر +à¸Ł ัà¸Ļ +ĠÑĢе ÑĨеп +ĠÑĢеÑĨеп ÑĤ +ĠبÙĨ Ùģس +ÑĢо ÑĪ +ĠмаÑĢ ÑĤа +Ġson ras +Ġsonras ı +×ķ×ij ש +ãĥª ãĤ¹ãĤ¯ +ĠFranç ais +á» ļ +ê° Ķ +Ġ×Ķ×ijר ×Ļת +פ ×Ļצ +פ×Ļצ ×ķ×Ļ +ĠÙĦÙħا ذا +ĠÐļи ев +ĠÑģ мÑĭÑģл +ê¸Ī ìľµ +ãĤ·ãĥ£ ãĥ« +ãĥ© ãĤ¤ãĥĪ +ìĽ ĥ +×ŀ ×Ĺר +ãĨ į +Ġkullan ım +Ġ×IJצ׾ ׳×ķ +Ġt Ãłn +ãĥı ãĥ¼ +ãģ¨ ãģ¨ãĤĤ +ãģ¨ãģ¨ãĤĤ ãģ« +ÑĢ ег +ÑĢег и +ÑĢеги он +ãģªãģı ãģªãĤĭ +Ġch ảy +Ġج ÙĩØ© +ÅĦsk iej +à¸Ńี à¹Ģม +à¸Ńีà¹Ģม ล +ãģį ãģ£ãģ¨ +ĠìĺĪ ìĤ° +Ġkit abı +Ġedu cação +Ġbul uÅŁ +олог иÑı +Ġкон кÑĢ +ĠконкÑĢ еÑĤ +×Ĵ ×Ļר +ĠпÑĢед лаг +ĠпÑĢедлаг аеÑĤ +ĠY ên +Ġíķľ ë²Ī +Ġ×ŀ ר׼×ĸ×Ļ +à¹Ģà¸Ľà¸´à¸Ķ à¹Ģà¸ľà¸¢ +ÑĤвеÑĢ д +ĠH á»ĩ +ĠÐĵ ÑĢ +à¸Ŀ à¹īา +×Ķ שק +×Ķשק ×¢×Ķ +Ġна Ñĥк +ìłIJ ìĿĦ +Ġн елÑĮ +ĠнелÑĮ з +ĠнелÑĮз Ñı +г ин +ĠB öl +ĠBöl ge +Ġв ла +Ġвла ÑģÑĤи +à¹Ģà¸Ļ à¹ĩ +à¹Ģà¸Ļà¹ĩ à¸ķ +ê³ ¨ +Ġö ld +Ġöld ür +׼׳ ×¢ +ĠاÙĦÙĩ ÙĬئة +ت ارÙĬØ® +ĠÐij ÑĢ +ĠÑģ мож +ĠÑģмож еÑĤе +ĠL úc +à¹Ħà¸Ľ à¸ĸึà¸ĩ +ĠBakan ı +Ġerklä rt +ĠÐIJ на +Ġsc ène +åķı ãģĦ +åķıãģĦ åIJĪãĤıãģĽ +ÙħÙĩ ÙĨد +ÙħÙĩÙĨد س +Ġн азвание +ив аниÑı +ãĤĴ å¤īãģĪ +ä»ĺãģį åIJĪ +ãĥij ãĤ½ +ãĥijãĤ½ ãĤ³ãĥ³ +æĺİ ãĤī +æĺİãĤī ãģĭ +à¹Ģà¸Ńà¸ģ สาร +à¹Ģà¸ģิà¸Ļ à¹Ħà¸Ľ +л еп +ãģĹãģŁ ãĤĤãģ® +ĠC âm +ĠCâm ara +ק×ķ׾ ׳×ķ×¢ +Ġ×ij×Ĵ ×Ļף +Ġoc zy +Ġoczy wiÅĽcie +att ivitÃł +ãĥĵ ãĥ¥ãĥ¼ +Ġeduc ación +Ä° YE +ê¹Į ìļĶ +ãĤ¨ ãĥªãĤ¢ +н еÑģÑĤи +Ġm óg +Ġmóg ÅĤ +Ġק×ĺ ׳×Ļ×Ŀ +ĠPr ä +Ġ×ľ×¢ ×ij×ķר +بÙĨ Ùī +з ол +зол оÑĤ +Ġwn ÄĻtr +ĠwnÄĻtr z +Ġconstr ução +รัà¸ļ รà¸Ńà¸ĩ +س جÙĨ +Ġק ×ķ׳ +ס ×Ļפ×ķר +ĠÙħ دÙī +رض Ùī +п лав +ï¼ ¥ +Ġil a +Ġila ç +ãĤĭ ãģ¹ãģį +ĠÙħ ÙĪÙĤÙģ +à¸ģร ุ +à¸ģรุ à¸ĵา +chodzÄħ c +ĠÑĤÑĭ Ñģ +Ðķ вÑĢо +ĠÙĬ ØŃدث +ãĥ¡ ãĤ¤ãĥ³ +ĠاÙĦص ØŃÙĬ +ĠÐĶ ан +دع اء +ãĤ´ ãĥ¼ãĥ« +ש ×ł×ª×Ļ +×©×ł×ª×Ļ ×Ļ×Ŀ +à¸Ķà¹īวย à¸ģัà¸Ļ +Ġol acaģı +Ġ×ij ×ŀ×Ĺ×Ļר +×Ķ ק +×Ķק ×ŀת +ãĥ¢ ãĥİ +ĠçalÄ±ÅŁ tı +Ġjó venes +ãģĦãģı ãĤī +ĠÙħ عدÙĦ +ĠC Å©ng +ĠSeg ún +Ġdönem de +Ġ׾ ×Ļ×ĵ×Ļ +ãģį ãģ¡ +ãģįãģ¡ ãĤĵ +ãģįãģ¡ãĤĵ ãģ¨ +Ùģر ÙĨس +ÙģرÙĨس ا +åIJij ãģį +Ġcamp aña +ĠÑģам оÑģÑĤоÑı +ĠÑģамоÑģÑĤоÑı ÑĤелÑĮно +á» Ģ +ÙĤ ÙĪا +س ÙĦاØŃ +à¸ģระ à¹ģ +à¸ģระà¹ģ ส +ĠполÑĮз Ñĥ +n qu +nqu ête +รà¹Īวม à¸ģัà¸ļ +ëĬIJ ëĥIJ +à¸Ĺีม à¸Ĭาà¸ķิ +Ġyıll ık +ìĬ ¬ +ĠØ£ صØŃاب +ill é +Ġdó la +Ġdóla res +Ġк ож +Ġкож и +ล à¹īà¸Ń +à¹Ģรีย à¸ļร +à¹Ģรียà¸ļร à¹īà¸Ńย +à¹Ģà¸ŀ ิ +à¹Ģà¸ŀิ à¹Īà¸ĩ +ÑĢиÑĤоÑĢ и +Ġí ijľ +Ġíijľ íĺĦ +ĠпеÑĢ ев +ĠпеÑĢев од +פ×Ĵ ×Ļ×¢×Ķ +ĠdeÄŁerlendir me +Ùģ ائ +ĠвÑĭ год +ınız ı +×ķ׼ ×Ļ×Ĺ +ĠдоÑģÑĤ иг +Ġng Ãłn +æĢĿ ãģ£ãģŁ +ĠÐķ ÑģÑĤÑĮ +ĠاÙĦر غÙħ +ĠzwiÄħz ane +رب Ø· +à¸Ļ ึà¸ĩ +Ġ׾×Ĺ ×ķק +Ġszczeg óln +Ġszczególn ie +Ġبا ستخداÙħ +ĠfÃŃs ico +×¢ ס +עס ×ķק +سÙĦ ÙĪÙĥ +Ġا ØŃد +Ñĩ ÑijÑĤ +×ĸ׼ ×Ķ +Ġl á»ĩnh +ĠÙĪ ØŃت +ĠÙĪØŃت Ùī +à¸Ħวาม สามารà¸ĸ +à¸Ńยูà¹Ī à¹ģลà¹īว +à¸ģาร à¹Ģà¸Ķิà¸Ļà¸Ĺาà¸ĩ +تخ Ø° +צ×Ļ ×ķ×ĵ +ĠاÙĦØ£ س +ĠاÙĦأس ÙĩÙħ +Ġt á»ĩ +ãģ£ãģ¦ ãģĦãģ¦ +สร ุ +สรุ à¸Ľ +Ġком ÑĦ +ĠкомÑĦ оÑĢÑĤ +ìĺ¤ ëĬĶ +ĠÑĢаз в +ĠÑĢазв ива +л анд +h änge +ĠبÙĨ سبة +à¹Ģà¸Ĥ ียว +עצ ×Ŀ +Ġ׾ ×ľ×Ľ×ª +Ñģо ÑĨиалÑĮн +Ġëĭ¤ìĿĮ ê³¼ +Ġרש ×ķ×ŀ +×ŀר ×Ĺ×ij +س ÙĤØ· +Ġalan ı +ĠÄij á»ĩ +é£Łãģ¹ ãĤĭ +à¸Ķ ึà¸ĩ +Ġgegen über +ĠبÙĩ Ø°Ùĩ +à¸ĸืà¸Ń à¹Ģà¸Ľà¹ĩà¸Ļ +ëķ ħ +à¸Ħà¸Ļ à¹Ħà¸Ĺย +ãĤ¢ ãĤ¦ +ãĤ¢ãĤ¦ ãĥĪ +ศ ัà¸ģ +ศัà¸ģ à¸Ķิ +ศัà¸ģà¸Ķิ à¹Į +ÙĤÙĪ اÙĨ +ÙĤÙĪاÙĨ ÙĬÙĨ +Ġhá»Ļ p +ãģªãģıãģª ãģ£ãģ¦ +Ġ×IJ ×ŀ׳ +Ġ×IJ×ŀ׳ ×Ŀ +à¹Ģà¸ķ ืà¸Ńà¸Ļ +ĠзавиÑģ им +ĠзавиÑģим оÑģÑĤи +ת ×Ļ×IJ +ת×Ļ×IJ ×ķר +å§ĭãĤģ ãģŁ +Ġng á»į +Ġngá»į t +íĴ į +ê³¼ ìŀ¥ +Ġb ại +ãģ§ãģį ãģ¦ +Ġcomeç ar +à¸Ľà¸£ าà¸ģ +à¸Ľà¸£à¸²à¸ģ à¸ı +Ġгод Ñĭ +м еÑģ +ĠاÙĦÙħست ÙĪÙī +ĠÑģам Ñĭе +л леÑĢ +ãģ£ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĻ +ãģ¨ãģ® ãģĵãģ¨ +bi ó +à¸ģล à¹Īà¸Ńà¸ĩ +ĠاÙĦز ÙĪج +ãģ«è¡Į ãģ£ãģŁ +à¸Ħà¹Ī à¸Ńà¸Ļ +à¸Ħà¹Īà¸Ńà¸Ļ à¸Ĥà¹īาà¸ĩ +ĠbaÄŁ l +ĠbaÄŁl ant +ĠbaÄŁlant ı +確 ãģĭ +確ãģĭ ãģ« +ãĥľ ãĥ¼ãĥ« +çµĤ ãĤıãĤĬ +ש ×ŀר +à¸Ĺีà¹Ī สามารà¸ĸ +ÙĦ زÙħ +д аеÑĤÑģÑı +รัà¸ļ à¸Ľà¸£à¸° +รัà¸ļà¸Ľà¸£à¸° à¸Ĺาà¸Ļ +å¤ī ãĤıãĤĬ +ï¼ ¢ +ĠìĺĪìĪĺ ëĭĺ +ãĤĪãģĨ ãģ¨ +มัà¸ģ à¸Īะ +ĠH Æ°Æ¡ng +ÙĨ ÙģØ° +×ŀ×ĵ ×ĵ +ĠìĿ¸ ìłķ +Ñħод иÑĤÑĮ +ĠзавиÑģ иÑĤ +×ķ×ĵ ×Ļ×¢ +ãģĵãģ¨ãģĮ ãģĤãĤĬãģ¾ãģĻ +ع راÙĤ +سط ØŃ +à¸ģำ à¹Ħร +ëĵ¤ ëıĦ +×Ļצ ×Ļר×Ķ +ãģĨ ãģĵãģ¨ +ÙĦا ØŃÙĤ +ãģĦ ãĤĮãģ° +ĠиÑģполÑĮз ÑĥÑİÑĤ +ĠB ợi +Ġשק׾ ×Ļ×Ŀ +ÑĨи кл +ÐIJ Ðŀ +Ġ×ijש ׳×Ķ +ÙĨØ´ Ø· +Ġש ×Ļ׳×ķ×Ļ +Ġש×Ļ׳×ķ×Ļ ×Ļ×Ŀ +Ġpobl ación +ĠH Æ°ng +ระ ว +ระว ัà¸ĩ +رÙĬاض Ø© +ر صد +تÙĤ ÙĦÙĬ +تÙĤÙĦÙĬ د +Ġülk em +Ġülkem iz +à¸Ĭ ะ +ãĤ¯ãĥª ãĥ¼ãĥł +èģŀ ãģĦãģŁ +Ġwa ż +Ġważ ne +ê±° ëĵł +ê±°ëĵł ìļĶ +×ŀ×IJ ×ijק +×Ĺ×ĵ ש×ķת +ĠW roc +ĠWroc ÅĤaw +ĠKü ltür +s ist +sist ência +×¢×ĸר ×Ķ +Ġg Æ°Æ¡ng +รà¹īาà¸Ļ à¸Ħà¹īา +ĠÙĪØ£ ÙĪضØŃ +ánd ose +ãĤ· ãĥ¼ãĥ³ +×IJ׳ ר×Ĵ +×IJ׳ר×Ĵ ×Ļ×Ķ +ãģªãģĦ ãģ§ãģĻ +Ġkh ủng +Ġ문 ìĦľ +Ġ×ij ×ĵ×ijר +×ĵ ×Ļ×ķ +×ĵ×Ļ×ķ ×ķ×Ĺ +Ġré gl +ÙħÙĪ اد +об оÑĢ +обоÑĢ оÑĤ +Ġ×Ķ ×ij׾ +Ġ×Ķ×ij׾ ×ķ×Ĵ +ØŃ Ø§Ùħ +ĠاÙĦع اص +ĠاÙĦعاص ÙħØ© +пеÑĢ аÑĤоÑĢ +ت Ø®ÙĦ +تخÙĦ ص +ãģŁãģł ãģĹ +ت سÙħ +à¹Ĥรà¸ĩ à¸ŀ +à¹Ĥรà¸ĩà¸ŀ ยา +à¹Ĥรà¸ĩà¸ŀยา à¸ļาล +ĠY ük +ĠYük sek +Ġש ׳×Ļת +Ġש׳×Ļת ף +liÄŁ e +Ġפ ת +Ġפת ×ķ×Ĺ +Ġbe ÄŁ +ĠbeÄŁ en +Ġ×ŀ ×ķר +Ġ×ŀ×ķר ׼×ij +Ġرس اÙĦØ© +íĨµ ìĭł +Ġaval ia +Ġavalia ções +Ġman h +Ġmanh ã +Ġìķ ŀ +Ġìķŀ ìľ¼ë¡ľ +ÙĤ تر +ÙĤتر ØŃ +à¹Ģà¸ģ ืà¸Ń +à¹Ģà¸ģืà¸Ń à¸ļ +Ġpropos é +Ø£ Ùħا +Ø£Ùħا ÙĥÙĨ +ĠÐŀ Ðŀ +ĠÐŀÐŀ Ðŀ +ÙħÙĤ ار +ÙħÙĤار ÙĨØ© +ëĦ IJ +ãģĦãģŁãģł ãģı +ÙĤ ÙĬÙĦ +Ġна ÑĪиÑħ +ãĤ« ãĥĥãĥĹ +×Ĺ׾ ת +Ġëĭ¤ ë§Į +à¸Ĺัà¹Īว à¹Ĥลà¸ģ +ãĥį ãĤ¿ +ØŃس اس +ãģ«ãģª ãĤĮ +ج ائ +جائ زة +é change +é conom +économ ie +Т Ðĺ +סת ׼׾ +à¸Ĺัà¹īà¸ĩ สà¸Ńà¸ĩ +ĠاÙĦØ® اÙħ +ĠاÙĦخاÙħ س +ק ×ĺ×¢ +au waż +à¸ľà¸¹à¹ī à¸Ĭาย +à¹ģà¸Ľà¸¥ à¸ģ +åIJĮæĻĤ ãģ« +зн аниÑı +ãģĦãģŁãģł ãģįãģ¾ãģĹãģŁ +Ġ×ŀ×ij ׾×Ļ +à¸Ĥà¸Ń à¹ĥหà¹ī +ĠاÙĦت ربÙĬØ© +Ġdécou vert +Ġżyc iu +apr ès +Ġy ab +Ġyab anc +Ġyabanc ı +ĠbaÅŁ layan +ìĹĪ ëįĺ +Ġhes abı +Ġë§Į ìķ½ +ë§ Īëĭ¤ +ĠTh ánh +ãĥ´ ãĤ¡ +à¸Ľà¸£à¸±à¸ļ à¸Ľà¸£ +à¸Ľà¸£à¸±à¸ļà¸Ľà¸£ ุà¸ĩ +ĠM ặc +à¹Ģหà¸ķุ à¸ľà¸¥ +ĠÐij ез +Ġcapac itÃł +ÅĤe ÅĽ +ĠпÑĢе им +ĠпÑĢеим ÑĥÑīеÑģÑĤв +ĠÅļ wiÄĻt +Ġpubli é +×ŀ×¢ צ×ij +Ùħشار Ùĥات +à¸łà¸² ษ +à¸łà¸²à¸© ี +Ġdeux ième +ĠÙħØŃ Ø§Ùģظ +ĠÙħØŃاÙģظ Ø© +ĠSch ön +ï½ ¤ +Ġ×Ķ ×ij×¢ +Ġ×Ķ×ij×¢ ×Ļ×Ķ +ĠÙĪاÙĦ ÙĦÙĩ +è¨Ģ ãģ£ãģŁ +à¸ķ à¹īาà¸Ļ +วร รà¸ĵ +à¸Ĺิ ศ +ĠbaÅŁ ına +Ġmog ÄĻ +ש ×Ļפ×ķר +ĠÙĪ عد +ĠÙĪعد Ùħ +Ġhistó rico +Ġk ısı +ĠìĿ´ ê²Į +ĠPol ÃŃtica +ĠÑģиÑĤÑĥ аÑĨии +ĠkoÅĦ ca +×ij×ĵ ×Ļק×Ķ +ĠاÙĦسÙĬ ارات +ãģªãĤī ãģ° +ãĤµ ãĥ© +ãĤĭãģĵãģ¨ãģĮãģ§ãģį ãĤĭ +Ġdecis ão +×ķ ×ķ×ĵ +lä ss +läss ig +Ġ׾ ×Ļשר×IJ׾ +ĠÙĬ أتÙĬ +ר ×ķ×ĸ +ö ÄŁ +Ã¶ÄŁ ret +Ã¶ÄŁret im +Ġд ек +Ġдек аб +Ġдекаб ÑĢÑı +Ġש ×Ĺ×ķר +ãģ¦ãģıãĤĮ ãģŁ +عب ارة +Ġélect rique +ĠاÙĦتÙĨ ÙħÙĬØ© +جر Ùī +ĠìĪĺ íĸī +à¸Ĺ ู +ĠÑĢе алÑĮно +Ñģп оÑģоб +à¸Ħล à¹īาย +Ġس عÙĪد +ön ü +ĠÙģ ÙħÙĨ +تÙĥ ÙĪ +تÙĥÙĪ ÙĬÙĨ +ĠкаÑĩ еÑģÑĤво +ĠконÑĤ ак +ĠконÑĤак ÑĤ +Ġsöz leÅŁme +à¸Ń à¹īาà¸ĩ +Ġت ÙĪÙģ +ĠتÙĪÙģ ÙĬر +×Ķ×ĸ ×ĵ +×Ķ×ĸ×ĵ ×ŀ׳×ķת +ĠØ·ÙĪÙĬÙĦ Ø© +Ġtér mino +Ġ×IJ ×Ļפ×Ķ +ãĥĵ ãĥ« +ส à¹Ĥม +สà¹Ĥม สร +ĠاÙĦ اث +ĠاÙĦاث ÙĨÙĬÙĨ +ев иÑĩ +Ġopin ión +à¸Ľ วà¸Ķ +åı¤ ãģĦ +ร à¹Īา +ĠB iaÅĤ +ĠÑģÑĤ ал +ĠÑģÑĤал о +ó logo +ĠìķĦ ëĭĪëĭ¤ +Ġ×IJ ×Ļת +Ġ×IJ×Ļת ×ķ +à¹Ģหà¹ĩà¸Ļ วà¹Īา +à¸ļ ารà¹Į +çĦ ¼ +çĦ¼ ãģį +ĠìĿ´ìļ© ìŀIJ +ĠнекоÑĤоÑĢ Ñĭе +ks z +ksz taÅĤ +ksztaÅĤ c +ãĤŃãĥ£ ãĥĥãĤ· +ãĤŃãĥ£ãĥĥãĤ· ãĥ³ãĤ° +Ġro ÅĽ +ĠroÅĽ lin +ÑĢаж а +×ij׳×Ļ ×Ļ×Ķ +à¸Ľà¸£ สิ +à¸Ľà¸£à¸ªà¸´ à¸ķ +Ġgörd ü +×ŀ׳×Ķ ×Ļ×Ĵ +å¤īãĤı ãģ£ãģ¦ +Ġ×IJ ×Ķ +Ġ×IJ×Ķ ×ijת×Ļ +à¹Ģร à¹Īà¸ĩ +Ġön ünde +Ġê·¸ ëĥ¥ +пол иÑĤ +полиÑĤ иÑĩеÑģк +ãĥ¡ ãĥĩãĤ£ +ãĥ¡ãĥĩãĤ£ ãĤ¢ +ĠDet ay +ĠDetay lı +ĠاÙĦصÙģ ØŃØ© +à¸ģาร à¹Ģà¸ĩิà¸Ļ +Ġìµľ ê·¼ +׼ ש׾ +ï¼ © +вÑĪ его +íķĺ ìĭ¤ +ĠÐŃ ÑĤ +ĠÐŃÑĤ оÑĤ +ส ื +สื à¸ļ +Ġng ừng +ĠдокÑĥменÑĤ ов +дав аÑĤÑĮ +ĠاÙĦشخص ÙĬØ© +Ġצ ×¢×Ļר +در Ùĥ +س ØŃب +à¹Ħมà¹Ī à¸Ħà¹Īà¸Ńย +Ġ×Ķ×ŀק ×ķ×ŀ×Ļ +สัà¹Īà¸ĩ à¸ĭืà¹īà¸Ń +Ġê·¸ê²ĥ ìĿĦ +ãģĤãĤĭ ãģĦ +ãģĤãĤĭãģĦ ãģ¯ +×IJ×ķ×ĺ ×ķ×ij +×IJ×ķ×ĺ×ķ×ij ×ķס +к ÑĨион +ĠÐľ ожно +ãģı ãģł +ãģıãģł ãģķ +ĠинÑĦоÑĢм аÑĨиÑı +ï» Ł +Ġìŀij ìĹħ +Ġ×Ļ ×ķסף +Ø¥ دارة +ĠاÙĦØŃ Ø§Ø¬ +×ł×¡ ×Ļ×¢×Ķ +из аÑĨиÑı +×IJ׾ ×ij +×IJ׾×ij ×ķ×Ŀ +п ед +Ġק×ĺ ׳×Ķ +ĠÙĨÙģس Ùĩا +ĠMinist ério +Ġп ен +Ġпен Ñģи +ãĥIJ ãĥ©ãĥ³ãĤ¹ +Ġ×Ķת ×ķר×Ķ +Ġt ạm +ĠìĹŃ ìĭľ +ï½ ¡ +Ġth á»± +Ġ ısı +ì» ¨ +ãģĹãģ£ãģĭãĤĬ ãģ¨ +Ġx Æ°a +Ġc ặp +×Ĺ ×Ļ×ij×ķר +วัà¸Ĵà¸Ļ à¸ĺรรม +st är +stär ke +ĠÑģам Ñĭй +p isa +pisa Äĩ +ĠoluÅŁ an +ĠاÙĦØ¥ ÙħاÙħ +ĠcÄĥ ng +Ġgü nl +Ġgünl ük +Ġ׳ש ×IJר +Ġkhi á»ĥn +ç¶ļ ãģijãĤĭ +stit ución +Ġcapac ité +Ġj aki +Ġjaki ÅĽ +вÑĪ иÑģ +вÑĪиÑģ ÑĮ +פע×ķ׾ ×ķת +ĠØŃ ÙĬات +ĠØŃÙĬات Ùĩ +Ġник огда +ÐĽ Ь +Ġ×Ķ×¢ ×ķ×ij +Ġ×Ķ×¢×ķ×ij ×ĵ×Ķ +Ġch Ãło +หลาย à¹Ĩ +ĠÑı н +ĠÑıн ваÑĢ +ĠÑıнваÑĢ Ñı +à¸Īำà¹Ģà¸Ľà¹ĩà¸Ļ à¸ķà¹īà¸Ńà¸ĩ +Ġhö her +ãģķãĤĮãģ¦ ãģĦãģŁ +สà¸ĩ สั +สà¸ĩสั ย +ĠاÙĦ اس +ĠاÙĦاس ÙĦاÙħ +ĠاÙĦØ´ Ùħس +สà¸ĸาà¸Ļ ี +ãĤ¯ãĥ© ãĤ¹ +à¸ŀร ร +à¸ŀรร à¸Ħ +p õ +põ e +Ġpor ém +à¸Ľà¸£à¸° สà¸ĩ +à¸Ľà¸£à¸°à¸ªà¸ĩ à¸Ħà¹Į +powied zie +powiedzie Äĩ +Ġмог Ñĥ +Ġж ел +Ġжел ез +ĠاÙĦØ« ÙĤ +ĠاÙĦØ«ÙĤ اÙģÙĬ +ĠпÑĢав ило +Ġgdy ż +פש ×ķ×ĺ +ÑĢабоÑĤ ка +ĠÙĥ رة +Ø´ دد +Ùħار Ùĥ +Ùħ ÙĥØ© +Ġпод пиÑģ +×ĺ×ķ ×ķ×Ĺ +ĠÅĽ c +ĠÅĽc ian +Ġر جاÙĦ +Ġ×ª×ľ ×ķ×Ļ +и ÑĪ +иÑĪ ÑĮ +Ġmé dec +Ġmédec in +ëįĶ ëĿ¼ëıĦ +ĠÑĤеб Ñı +Ġ׾×Ķ ×ķס×Ļ×£ +ãģĬ 話 +Ġà¹ģà¸ķà¹Ī à¸ģà¹ĩ +د اÙģ +داÙģ ع +ĠC ùng +ãĥ»ãĥ» ãĥ»ãĥ» +ê¶ ģ +Ġdeber ÃŃa +หà¸Ļà¹Īวย à¸ĩาà¸Ļ +Ġva ÌĢ +Ġעצ ×ŀ +Ġעצ×ŀ ×Ŀ +à¹Ģà¸Ĭืà¹Īà¸Ń วà¹Īา +שק ×¢ +Ġ×Ķ ׼×ķ׾ +Ġ×Ķ׼×ķ׾ ׾ +ни бÑĥд +нибÑĥд ÑĮ +ĠëĦĪ íĿ¬ +Ġоб ÑĢаÑī +ĠобÑĢаÑī а +Ġ×¢×ij×ķ×ĵ ת +ĠاÙĦÙħÙĨت خب +ıy ord +ıyord u +ÙĪ Ø° +×Ĺש ×Ļ×ij×ķת +Ġ×Ķ×¢ ×Ļק +Ġ×Ķ×¢×Ļק ר×Ļ +ì¢ Į +ยุ à¹Ĥร +ยุà¹Ĥร à¸Ľ +Ġа пÑĢ +ĠапÑĢ елÑı +sz ed +szed ÅĤ +д он +à¹Ģà¸ķิ à¸ļ +à¹Ģà¸ķิà¸ļ à¹Ĥà¸ķ +кол о +Ġkażde j +å¸ ° +帰 ãĤĬ +Ġмил ли +Ġмилли он +ç¾İåij³ ãģĹãģĦ +ت ÙĤار +تÙĤار ÙĬر +ĠìĿ´ 루 +ĠìĿ´ë£¨ ìĸ´ +Ġsprzeda ż +×Ķ ×ķצ×IJ×ķת +ãĤ¢ãĤ¯ ãĤ» +ãĤ¢ãĤ¯ãĤ» ãĤ¹ +ר ×ķ×¥ +ĠгоÑģÑĥдаÑĢÑģÑĤв енн +Ø£ ØŃÙĥ +Ø£ØŃÙĥ اÙħ +ĠoluÅŁ u +ĠA ç +ĠAç ık +ãĤ¸ ãĥ¼ +ç´ł æĻ´ +ç´łæĻ´ ãĤīãģĹãģĦ +Ġ×ijש×ij ×ķ×¢ +ب Ø° +بذ ÙĦ +สา à¹Ģหà¸ķุ +Ġpoz osta +Ġpozosta ÅĤ +ØŃر Ùħ +Ġimport ância +leÅŁtir me +Ġд ÑĢев +Ġmó vil +ĠA ynı +Ġна лог +Ġналог ов +Ġ×Ĺ ×Ļפ×Ķ +ĠÑĦоÑĢм Ñĥ +à¸Ĺà¸Ķ สà¸Ńà¸ļ +ĠksiÄħż ki +Ġma ÅĤe +Ùħس Ø£ÙĦ +ÙħسأÙĦ Ø© +ï¼¾ ï¼¾ +ç ãeste +év iter +Ġкон ÑģÑĤÑĢÑĥк +ĠконÑģÑĤÑĢÑĥк ÑĨи +ï¾ ŀ +Ġת×ķ׼ ׳ +ãĤ¹ãĥĪ ãĥ¬ãĤ¹ +ĠاÙĦاÙĤتصاد ÙĬ +×ŀ×ĵ ×Ļ +Ġw ÅĤad +ĠwÅĤad z +Ø® ÙĪÙģ +ĠмаÑĤеÑĢиал ов +ãģ¨ãģ£ãģ¦ ãĤĤ +Ġznaj du +Ġznajdu jÄħ +Ùģ ئة +ãģ©ãģ® ãĤĪãģĨãģª +æĬij ãģĪ +׳ ×Ĺ׾ +Ġdü ny +Ġdüny an +Ġdünyan ın +гÑĢ ани +гÑĢани Ñĩ +Ġ×Ķש׾ ×Ļש×Ļ +Ġ×Ķ×IJ ש +åıĬ ãģ³ +ìĭŃ ìĭľ +ìĭŃìĭľ ìĺ¤ +Ġдол л +Ġдолл аÑĢ +Ġпов ÑĤоÑĢ +Ġ×Ĺ ×Ļ׳×Ŀ +ת פת×Ĺ +Ñĥв ели +Ñĥвели Ñĩен +ãĤ« ãĥª +raw id +rawid ÅĤow +×ķ ×ķ׾ +ãĥŁ ãĥ¥ +ì½ ĺ +ĠBy ÅĤ +Ðľ ÐIJ +ع ÙIJ +ĠÑģовеÑĢ ÑĪ +ĠÑģовеÑĢÑĪ енно +Ġм ой +Ġ×ķ׾×IJ ×Ĺר +æħ £ +æħ£ ãĤĮ +ØŃ Ø§Ùģظ +Ġ무 ë£Į +à¸Ħà¸ĵะ à¸ģรรม +à¸Ħà¸ĵะà¸ģรรม à¸ģาร +Ġìĸ´ ëĶĶ +Ġdif eren +Ġdiferen ça +ĠاÙĦØ£ ساس +ĠاÙĦأساس ÙĬØ© +Ġ׾×IJ×Ĺר ×ķ׳×Ķ +ê· ł +Ġ×Ķש׳×Ļ ×Ļ×Ķ +ìľĦìĽIJ ìŀ¥ +ลุ à¸ģ +ç iler +Ġ×Ķ×IJ ׾×ķ +èģŀ ãģı +Ġ×ķ×IJ פ×Ļ׾×ķ +ĠÑĢе ализ +ĠÑĢеализ аÑĨи +ระยะ à¹Ģวลา +Ġجدا Ùĭ +تب اع +Ġveh ÃŃculo +Ġдол г +à¸Ľà¸£à¸´ มาà¸ĵ +ì¦ IJ +Ġ׾ ×ŀק×ķ×Ŀ +ĠìĤ¬ ì§Ħ +à¸Ĭ à¹īา +Ġ×ŀ×¢ ×ķ׾×Ķ +Ġgö rm +Ġgörm ek +ĠÙĪÙĩ Ø°Ùĩ +пеÑĢ в +пеÑĢв ÑĭÑħ +ê·¸ ëŀĺ +ĠاÙĦبر ÙĬØ· +ĠاÙĦبرÙĬØ· اÙĨÙĬ +ĠиÑİ Ð½Ñı +ĠÐĵ оÑĢ +Ġ׾ ש׾×Ŀ +ÐIJ ÐĿ +Ġназ наÑĩен +о оÑĢ +ооÑĢ Ñĥж +Ġöz elli +Ġözelli ÄŁi +Ġни же +ç¶ļ ãģijãģ¦ +Ġа ÑĢенд +Ġkat ılı +Ġkatılı m +ĠØ¥ Ø·ÙĦاÙĤ +ĠÙĪØ¥ ذا +Ġок ÑĤÑı +ĠокÑĤÑı бÑĢÑı +à¹Ĥà¸ķ ๠+à¹Ĥà¸ķ๠Ĭ +à¹Ĥà¸ķà¹Ĭ ะ +Ġolduk ları +Ùħ ÙĪÙĤع +ëĤ © +ãģ¨æĢĿ ãģ£ãģ¦ãģĦãĤĭ +Ġש ×Ļ׼×ķ׾ +วา à¸Ķ +س ÙĬÙĦ +à¸Ĥ วั +à¸Ĥวั à¸į +تØŃ ÙĥÙħ +ì ĤŃ +Ġconna ît +׳ פת×Ĺ +Ġch ặ +Ġchặ n +ĠÙħ ØŃÙħ +ĠÙħØŃÙħ ÙĪد +ãģ ´ +ĠпÑĢодÑĥк ÑĨии +зд ÑĢав +ãģĶ è¦ +ãģĶè¦ § +×IJ×ij ×IJ +Ġvé ritable +ĠØ· ÙģÙĦ +ãĥĪãĥ© ãĥĸãĥ« +ê³ ¡ +Ġת ×ŀ×ķ׳×Ķ +Ġki ên +ĠÙĤ ادر +Ø¥ÙĤ ÙĦÙĬÙħ +ĠпÑĢед пÑĢи +ĠпÑĢедпÑĢи ÑıÑĤиÑı +Ġb Äĥng +Ġay ında +Ġg ấp +еÑħ ал +Ġgi Ãłnh +Ġд ав +Ġдав но +ìĺĢ ëĭ¤ +à¸Ļัà¸ģ à¹Ģà¸ķ +à¸Ļัà¸ģà¹Ģà¸ķ ะ +Ùħست شار +ست راتÙĬج +ستراتÙĬج ÙĬ +رÙħ ز +Ġt Ä©nh +ë¡ Ń +ĠÑĩ еÑĤ +ĠÑĩеÑĤ Ñĭ +ĠÑĩеÑĤÑĭ ÑĢе +ĠEnt ão +Ġص غ +Ġصغ ÙĬرة +×ij×Ļ×ĺ ×ķ׾ +خط ÙĪØ· +ĠÑĢазвиÑĤ ие +Ġamacı yla +à¸Ĺี วี +Ġо ÑģÑĤ +ĠоÑģÑĤ алÑĮн +ש×ķ׾׊ף +Ġ׼ ׳×Ļס +Ġ׼׳×Ļס ×Ķ +Ġd áºŃy +ĠyaÅŁ ayan +Ġ×ŀ×Ķ ×ķ×ķ×Ķ +ĠÑĥ Ñģи +ĠÑĥÑģи ли +×ŀ פ×Ļ +ĠпÑĢовед ениÑı +Ġر ب +Ġرب Ùħا +ĠاÙĦØ£ ÙĪسط +Ġìľł ì§Ģ +Ġprac ownik +Ġpracownik ów +×ŀס ×ķרת +ÙĤار ب +à¸Ħวาม รูà¹īสึà¸ģ +à¹ģหล ะ +ĠاÙĦÙĨ ÙĤد +Ġ×IJ׾ פ×Ļ +Ùħس ئ +Ùħسئ ÙĪÙĦ +ев ÑĭÑħ +клÑİÑĩ ениÑı +×ij ×Ļ׳ +×ij×Ļ׳ ×Ļ×Ķ×Ŀ +ש ×ķ×IJ×Ķ +ĠÅŁ ark +ĠÅŁark ı +Ġsü rec +Ġsürec in +à¹Ģà¸Ħร à¸Ķ +à¹Ģà¸Ħรà¸Ķ ิà¸ķ +ãĥIJ ãĥ¬ +ĠØ´ Ø£ÙĨ +à¹Ģà¸Ńา à¹Ħวà¹ī +niÄĻ cie +רצ ×Ĺ +ĠaÅŁ ama +׳ פ×Ĵ×¢ +Ġth á»Ŀ +Ġkhu ẩn +diÄŁ inde +ÑıÑī иÑħ +ãĥĺ ãĥ« +Ġüber h +Ġüberh aupt +ĠÑĤÑĢеб ова +ĠdÅĤ ugi +×ĺ ×Ļף +à¸Ĥà¸Ļาà¸Ķ à¹ĥหà¸įà¹Ī +ĠاÙĦØ£ Ùĩ +ĠاÙĦØ£Ùĩ ÙĦÙĬ +ĠMü d +ĠMüd ürü +Ġ×Ļ×Ķ ×ķ×ĵ×Ķ +Ñĭв аеÑĤÑģÑı +س اط +×Ķת ׳×Ķ×Ĵ +×Ķ×ª×ł×Ķ×Ĵ ×ķת +à¸ģาร à¸ľà¸¥à¸´à¸ķ +íĴ Ģ +สà¸ĸาà¸Ļ à¸ģารà¸ĵà¹Į +Ġо ÑĦ +ĠоÑĦ иÑģ +ĠÙĦ عبة +Ġstron ÄĻ +Ġר×IJ ×ķ×Ļ +×Ĺ ×ij׾ +ĠÑĢÑĭ н +ĠÑĢÑĭн ке +Ġ׾×ŀ×¢ ף +اس ÙĦ +ห ัà¸Ļ +Ġ×IJ ×Ĺ×Ļ +ĠпÑĢод ол +ê°Ģ ìŀħ +Ġ×ijר ×Ĺ +Ġ×ijר×Ĺ ×ij×Ļ +дж еÑĢ +Ġ׾ ×Ĺ׾ +Ġ׾×Ĺ׾ ×ķ×ĺ +Ġ׾×Ĺ׾×ķ×ĺ ×Ļף +ศาส à¸Ļา +ãĤ¢ãĤ¤ ãĥĨ +ãĤ¢ãĤ¤ãĥĨ ãĥł +Ġפר ×ķפ +جز اء +ล à¸Ńย +Ġc iaÅĤa +Ġgi ết +ĠзнаÑĩ иÑĤелÑĮно +Ġolmad ıģ +Ġolmadıģ ını +н д +нд екÑģ +تأ Ùĥد +Ġìĸ ¸ +Ġìĸ¸ ìłľ +ay dın +ãĥī ãĥ¬ãĤ¹ +Ġs ắt +Ġíĺ¸ íħĶ +Ġë¶ ģ +Ġë¶ģ íķľ +ãĥij ãĤ¤ +Ġ×ŀש×Ĺק ×Ļ +à¸Ħà¸Ļ à¸Ńืà¹Īà¸Ļ +Ġиз гоÑĤов +ĠизгоÑĤов лен +à¹Ģà¸ģีย ร +à¹Ģà¸ģียร à¸ķิ +תק שר +ĠÑĢаÑģ ÑĩеÑĤ +ส à¹Ģà¸ķ +Ġl änger +ĠiÅŁ let +ĠiÅŁlet me +Ġع ÙĦÙĬÙĨ +ĠعÙĦÙĬÙĨ ا +é lection +ĠاÙĦغ ربÙĬØ© +íĭ Ģ +ãĤĤãĤī ãģĪ +Ġкни ги +Ø£ سÙħ +أسÙħ اء +Ġth á»ı +Ġthá»ı a +หà¸Ļ ู +Ġ×ł×¢ ש×Ķ +à¸łà¸²à¸¢ à¹ĥà¸ķà¹ī +à¸ŀื à¸Ĭ +رÙĬ Ø· +Ùģ ÙĪض +ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ ãģĦãģ¾ãģĹãģŁ +ש ×ĵ×Ķ +Ġng á»±c +ĠÑģеÑĢ ÑĮ +ĠÑģеÑĢÑĮ езн +T ôi +Ġfiyat ları +ĠвÑģ Ñİ +ĠC ódigo +Ġ×Ķש ×IJ +Ġ×Ķש×IJ ׾×Ķ +ĠP ública +Ø¥ Ø® +إخ ÙĪاÙĨ +ĠзаÑıв ил +ãĥ¦ ãĥ¼ +ר×IJ ×Ļת +vol ución +Ġsz ko +Ġszko ÅĤy +جرÙĬ دة +Ġpens é +ìī ¬ +ĠBüyük ÅŁehir +ĠØ£Ùħ رÙĬ +ĠØ£ÙħرÙĬ ÙĥÙĬ +à¸Ļัà¸ģ ศึà¸ģษา +Ġtod av +Ġtodav ÃŃa +ĠС ан +ĠСан кÑĤ +íķĺ ìŀIJ +ØŃÙĪ اÙĦ +׼ ×ķשר +à¹Ģลย à¸Ħรัà¸ļ +Ġal gu +Ġalgu ém +Ùģ ز +Ġçek il +Ġ×ĵ ר׼×Ļ×Ŀ +ãĥIJ ãĥ© +à¸ģà¹ĩ สามารà¸ĸ +สà¹Īวà¸Ļ ลà¸Ķ +íı ° +ĠP úb +ĠPúb lico +à¹ģà¸Ļว à¸Ĺาà¸ĩ +×IJת ×Ĵר +Ø´ اش +شاش Ø© +ci ÅĽni +ĠÃľ rün +ÙĦÙĪ ØŃ +ĠاÙĦ بÙĨ +ĠاÙĦبÙĨ Ùĥ +ì¡° ì¹ĺ +Ġorganiz ación +ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ ãģĦãģ¾ãģĻ +s ätze +ĠÑģем ей +ÙĤ صد +ÑģÑĤв еннÑĭе +Ġpréc éd +Ġprécéd ent +à¸ģรุà¸ĩà¹Ģà¸Ĺà¸ŀ ฯ +ãģ¨è¨Ģ ãģĦ +×ij׳×Ļ ×Ļף +ĠØŃ ÙĪ +ĠØŃÙĪ اÙĦÙĬ +סק ס +ĠsaÄŁlam ak +Ġ׾ צ×Ļ×Ļף +ק×ĵ ש +Ġ×Ķ×ŀ ×¢×¨×Ľ×ª +Ġ׾×Ķ ×¢×ij×Ļר +Ġg ünd +Ġgünd em +ĠнаÑĪ его +à¹ĥà¸Ļ à¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī +à¹Ģà¸Ħร ืà¸Ń +à¹Ģà¸Ħรืà¸Ń à¸Ĥ +à¹Ģà¸Ħรืà¸Ńà¸Ĥ à¹Īาย +ظ اÙĩرة +ÙħÙĨ ظÙħ +ÙħÙĨظÙħ ات +Ùħت از +追 ãģĦ +dı kt +dıkt an +ĠëįĶ ìļ± +ĠÐĿ апÑĢимеÑĢ +tw ór +×ŀ×ķ×¢ צ×Ķ +Ùĥ ÙĪÙĥ +Ð © +×ŀ×ĺ פ׾ +ó lica +訪 ãĤĮ +ĠëĮĢ ë¶Ģ +ĠëĮĢë¶Ģ ë¶Ħ +ãĤ¯ãĥª ãĥĥãĤ¯ +ãĤĴ éģ¸ +ãĤĴéģ¸ ãģ¶ +Ġpow sta +Ġpowsta ÅĤ +Ġraz ón +×ij ×ķ×Ĺר +ĠÑģообÑī ил +Ġק ×ij×ķ×¢ +r êt +à¸Ķี à¸Ĥึà¹īà¸Ļ +×ŀס ×¢×ĵ +×ŀסע×ĵ ×ķת +ĠÃĸ sterreich +Ġ׳ ×Ĺש×ij +Ùħباد رة +ì´ ī +×Ĵ ׳×ĺ×Ļ +ä¿¡ ãģĺ +du ÄŁ +duÄŁ unu +Ġph ú +ĠاÙĦØ£ Ø®ÙĬر +Ġت عتبر +landır ıl +ãģ¨ãģ¯ ãģĦ +ãģ¨ãģ¯ãģĦ ãģĪ +ĠاÙĦ Ø·ÙĦ +ĠاÙĦØ·ÙĦ اب +ĠN º +éģ¿ ãģij +اÙĦ Ùħع +اÙĦÙħع رÙĪÙģ +ส à¸łà¸² +éĽ¢ ãĤĮ +ĠпомоÑī ÑĮ +Ġзна еÑĤ +ãĥĹãĥ¬ ãĤ¼ +ãĥĹãĥ¬ãĤ¼ ãĥ³ãĥĪ +Ġsup érieur +Ġש׾ ×Ļש×Ļ +ĠاÙĦÙĨ ÙĪع +ãĤĵãģ§ãģĻ ãģŃ +à¸Ńà¸ļ รม +Ġgi á»įng +Ġwzgl ÄĻd +ĠاÙĦÙģ ÙĤر +è rent +Ġ×ŀ×IJ ×Ĺ +Ġ×ŀ×IJ×Ĺ ×ķר×Ļ +×Ĵ ×Ĵ +×Ļ ×Ļ×ij +ÙħÙĦ اب +ÙħÙĦاب س +Ġhük ü +Ġhükü met +Ġ×ŀ×Ĵ ×Ļ×ij +ĠÐŀ Ñĩ +ĠÐŀÑĩ енÑĮ +æĹ© ãģĦ +Ġconstr ucción +Ġth ượng +ï¼ ĭ +Ġcor ação +à¹Ģหล à¹ĩà¸ģ +ĠBaÅŁ b +ĠBaÅŁb akan +éĢ£ ãĤĮ +ãģĻãĤĭ ãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ +ĠÙĤ اÙħت +Ġا Ùĥثر +Ùģاع ÙĦ +ĠÑĦ оÑĢ +ĠÑĦоÑĢ Ñĥм +غ Ø°ÙĬ +ĠiÅŁ le +ĠiÅŁle ml +ĠiÅŁleml eri +ĠìĤ¬ëŀĮ ìĿĢ +Ġìŀij ìĦ± +Ġë§Ī 볨 +Ùħ جÙĦس +หม ู +д в +дв иг +двиг а +à¹Ģสีย à¸Ĭีวิà¸ķ +×Ķת פת×Ĺ +×Ķתפת×Ĺ ×ķת +ĠмеÑĤ ÑĢо +ĠÑģ енÑĤ +ĠÑģенÑĤ Ñı +ĠÑģенÑĤÑı бÑĢÑı +ê³ § +Ġ׾ פע +Ġ×ľ×¤×¢ ×ŀ×Ļ×Ŀ +à¹Ģà¸ļ ีย +詳 ãģĹãģı +çķ° ãģªãĤĭ +ĠÄ°l çe +ĠAt at +ĠAtat ür +ĠAtatür k +รุ à¹Īà¸ĩ +Ġkald ı +Ġ주 ìŀ¥ +Ġprés ence +Ġн аб +Ġнаб лÑİ +ĠнаблÑİ Ð´Ð° +ĠÑģам ого +×Ĵ ×ķש +×ŀ×ĺ ×ķפ +×ŀ×ĺ×ķפ ׾ +ĠвÑĭб иÑĢа +ĠìŀIJ 리 +åĪĨ ãģĭãĤīãģªãģĦ +Ġз Ñĥб +Ġש׼ ×ijר +Ġد ائ +Ġدائ Ùħا +ĠпаÑĢ ÑĤи +ï¼ ² +ĠاÙĬ ضا +ĠÑħ оз +ĠÑħоз Ñı +ĠÑħозÑı й +ĠÑħозÑıй ÑģÑĤв +ĠاÙĦØ£ ج +ĠاÙĦأج ÙĨب +ĠاÙĦأجÙĨب ÙĬØ© +ĠÐĹ Ð½Ð° +ĠAp ós +ĠÑį неÑĢ +ĠÑįнеÑĢ ги +Ġy ans +Ġyans ı +ĠJust i +ĠJusti ça +Ġpré vu +ม วล +ìŀ¥ ëĭĺ +à¸ģระ à¸ļ +à¸ģระà¸ļ วà¸Ļ +à¸ģระà¸ļวà¸Ļ à¸ģาร +×ŀ ×ŀ +×ŀ×ŀ ×ķצע +Ġh ẹ +Ġhẹ n +зд ание +Ġak ÅŁ +ĠakÅŁ am +×ĺ ×ķפ +Ġgere kt +Ġgerekt i +Ġgerekti ÄŁini +Ġnar z +Ġnarz ÄĻdzi +é po +épo que +ĠTh ần +Ġwys oko +Ġwysoko ÅĽci +à¸ľà¸¹à¹ī à¸Ľ +à¸ľà¸¹à¹īà¸Ľ à¹Īวย +ĠÙĬ بدÙĪ +ÑĤелÑĮ ного +Ġвз глÑıд +Ġjed nÄħ +ĠìĿĺ 견 +Ġ à¸Ĥà¸ĵะà¸Ĺีà¹Ī +פ ×Ļ×ĵ +ìĥģ ëĭ´ +Ġm ỡ +×Ķ ×ŀ׾ +×Ķ×ŀ׾ צ×ķת +ĠÑģоÑģÑĤ о +ĠÑģоÑģÑĤо иÑĤ +Ġав и +Ġави а +ĠL änder +تص ÙĪÙĬر +×ŀ×ĵ ×Ļ×Ķ +ìłĪ ì°¨ +ãģ¨ ãĤĬ +ãģ¨ãĤĬ ãģĤ +ãģ¨ãĤĬãģĤ ãģĪ +ãģ¨ãĤĬãģĤãģĪ ãģļ +ĠÑĢ Ñıд +ĠÑĢÑıд ом +ĠNh ất +ĠاÙĦÙĥ اÙħÙĦ +×Ĺ׾ ׾ +ĠGi ấy +צ ×ĺר +צ×ĺר ×£ +Ġ׾×ij ×ĺ׾ +Ġим еÑĤÑĮ +ס×ŀ ×ķ×ļ +Ġparticip ação +íķľëĭ¤ ë©´ +ÙħÙĨت دÙĬ +ÙħÙĨتدÙĬ ات +ĠeÄŁ len +g änge +رب ØŃ +ãĤ® ãĥ£ +ĠاÙĦر ÙĤÙħ +à¸ĭ à¹īำ +ĠH óa +×ŀר ×Ĺק +ØŃÙħ اÙħ +بÙĪ Ùĥ +ĠArt ÃŃculo +ãĥĦ ãĤ¢ãĥ¼ +×Ķפ ׼×Ķ +×Ĺ׾ ×ķף +ĠпеÑĢе Ñħод +len miÅŁ +زر اعة +Ġseñ or +ãģ£ãģ¦ ãģįãģ¦ +Ø¥ Ø´ +إش ارة +Ġpod ÃŃa +ĠÃľ lke +н ÑģкаÑı +Ġadapt é +Ġdüzen len +Ġdüzenlen en +ĠÑģÑĤ ала +ĠÙĬ ØŃتاج +Ġn ier +Ġnier uch +Ġnieruch omo +Ġnieruchomo ÅĽci +ãģĵãģ¨ãģĮ ãģĤãĤĭ +ยà¸Ńà¸Ķ à¹Ģยีà¹Īยม +ĠÙħ ج +ĠÙħج اÙĨÙĬ +Ġз аб +Ġзаб ол +Ġзабол ев +Ġзаболев аниÑı +ĠÅĽ ro +ĠÅĽro dk +ĠÅĽrodk ów +Ġ×Ķ ׾×IJ×ķ×ŀ×Ļ +Ġdok ÅĤad +ĠdokÅĤad nie +ãģŁãģı ãģªãģĦ +ãģ¯ãģļ ãģ§ãģĻ +ãģ¨æĢĿ ãģ£ãģ¦ãģĦãģŁ +é cran +ìĹħ ì²´ +trzym aÅĤ +ÑģÑĤв еннÑĭй +ĠNot ÃŃc +ĠNotÃŃc ias +Ùħ رÙĬ +ÙħرÙĬ ض +æ°Ĺ è» +æ°Ĺè» ½ +æ°Ĺ軽 ãģ« +ëĵ £ +Ġ×ĵ ×ķ×IJר +Ġ׾ ×ŀ׳ +Ġ׾×ŀ׳ ×ķ×¢ +ĠçalÄ±ÅŁ ıyor +ĠÅŁ idd +ĠÅŁidd et +ĠM ặt +Ġate ÅŁ +ĠполÑĥÑĩ ениÑı +à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ มืà¸Ń +Ġgrö ÃŁer +د ائ +دائ رة +Ġbul un +Ġbulun maktadır +à¹Ģห ร +à¹Ģหร ีย +à¹Ģหรีย à¸į +à¸Ļัà¸ģ à¸Ĺà¹Īà¸Ńà¸ĩà¹Ģà¸Ĺีà¹Īยว +Ġalan ında +ĠÑĥ зна +Ġл еÑĩение +売 ãĤĮ +Ġçev ir +Ġdeste ÄŁi +ĠheiÃŁ t +âĸ ² +ØŃ Ø· +à¸Ħำ à¸ķà¸Ńà¸ļ +ãĤªãĥ³ ãĥ©ãĤ¤ãĥ³ +Ġ×ij×Ĺ×Ļ ×Ļ×Ŀ +ãĥ¦ ãĥĭ +Ġdüzenle me +Ġmodal itÃł +سر Ø· +سرط اÙĨ +×ŀ׼ ×ķף +ĠданнÑĭ й +تر ت +ترت ÙĬب +à¸ļาà¸ĩ à¸Ħà¸Ļ +ĠÄIJ á»ĭnh +ม ูล +มูล à¸Ħà¹Īา +ÙĨ ÙĤص +à¸ģาร รัà¸ģษา +ĠÑĦ он +ĠÑĦон д +ãĤĪãģĨ ãģ«ãģªãģ£ãģŁ +Ùħع اÙĦ +ÙħعاÙĦ جة +ĠOs man +ĠOsman lı +иÑĩеÑģк ом +à¸Ńยาà¸ģ à¸Īะ +ãģķãģ¾ ãģĸ +ãģķãģ¾ãģĸ ãģ¾ +ãģķãģ¾ãģĸãģ¾ ãģª +Ġת ×ķ׼׾ +×¢ צ×ij +ĠاÙĦع سÙĥ +ĠاÙĦعسÙĥ رÙĬ +Ġvé hic +Ġvéhic ule +Ġ×Ļצ ×Ĺק +ĠاÙĦÙĪ ØŃ +ĠاÙĦÙĪØŃ ÙĬد +ĠاÙĦع دÙĪ +ĠQu ản +Ġê³µ ëıĻ +بد ÙĦ +ĠÄij ảng +Ġm á»ĩnh +Ġnie zb +Ġniezb ÄĻ +ĠniezbÄĻ dn +Ġyayın lan +обÑī и +Ġgö tür +צ פ +צפ ×ķ×Ļ +ĠÙĦÙĬ بÙĬ +ĠÙĦÙĬبÙĬ ا +ØŃ ÙĪا +Ġд об +Ġдоб ÑĢо +иÑĢÑĥ ем +ĠاÙĦØŃÙĥÙĪÙħ ÙĬØ© +m Ã¤ÃŁig +Ġed ición +влек аÑĤелÑĮ +влекаÑĤелÑĮ н +Ġת ש׾×ķ×Ŀ +Ġ×Ķש ×ķ׳×Ļ×Ŀ +มิ à¸ĸุ +มิà¸ĸุ à¸Ļ +มิà¸ĸุà¸Ļ ายà¸Ļ +é£Łãģ¹ ãģ¦ +ĠìĪĺ ì§ij +ס ×ij×Ļ +ĠиÑİ Ð»Ñı +Ġà¹Ħà¸Ķà¹ī à¹ģà¸ģà¹Ī +׾×Ĺ ×Ŀ +tr ä +trä gt +ãģĿãĤĤ ãģĿãĤĤ +ÐĿ Ðķ +Ġв нÑĥÑĤ +ĠвнÑĥÑĤ ÑĢи +ãģ¨ ä¸Ģç·Ĵãģ« +ãĤ« ãĥķãĤ§ +Ġ×ij×Ĺ ×ĵר +×Ĺ ×ŀש +ãĤ¨ ãĥį +ãĤ¨ãĥį ãĥ« +ãĤ¨ãĥįãĥ« ãĤ® +ãĤ¨ãĥįãĥ«ãĤ® ãĥ¼ +à¸Ĥà¸Ńà¸ĩ à¸ķัวà¹Ģà¸Ńà¸ĩ +بÙĤ اء +פס ×Ļ׼ +פס×Ļ׼ ×ķ׾×ķ×Ĵ +ãĥ¡ ãĥĥ +ãĥ¡ãĥĥ ãĤ» +ãĥ¡ãĥĥãĤ» ãĥ¼ãĤ¸ +ÙĦ ÙĤب +A Äŀ +שק ×Ļ×¢ +ÙĤ ساÙħ +×ĵ×ķ×Ĵ ×ŀ×Ķ +æ·± ãģĦ +íĸĪ ëĬĶëį° +ĠrozwiÄħz anie +à¸Ļัà¹Īà¸Ļ à¹Ģà¸Ńà¸ĩ +×Ļצ ×ij +Ġtr ông +à¹ĥà¸Ĭà¹ī à¸ļริà¸ģาร +ĠاÙĦÙħÙĪ سÙħ +ĠдеÑĤ и +ãģĹãģĭ ãģªãģĦ +ס ×Ļף +Ġréfé rence +à¹ģห à¹īà¸ĩ +ãĤĤãĤī ãģ£ãģŁ +Ġ׾ ר׼ +Ġ׾ר׼ ×ķש +شع ÙĪر +ĠÐij ог +Ġlaz ım +Ġ×Ļש ׳×Ŀ +Ġп аÑĢÑĤ +ĠпаÑĢÑĤ неÑĢ +ĠÑĥ ника +ĠÑĥника лÑĮн +Ġmaté riel +×ŀר ק +Ġph Æ°á»Ŀng +Ġз ай +Ġзай м +Ùģ ÙĤد +Univers itÃł +×¢ ר׼×Ļ×Ŀ +Ġba ño +Ġн оÑı +ĠноÑı бÑĢÑı +à¸Ľ à¹īาย +Ġt ats +Ġtats äch +Ġtatsäch lich +ĠÑĤÑĢ еÑĤÑĮ +Ñį м +ãĥĻ ãĥ¼ãĤ¹ +Ġnh á»±a +ìĬ¤ íģ¬ +ĠعبداÙĦ ÙĦÙĩ +Ġת ×ķר×Ķ +أش ÙĬ +أشÙĬ اء +ĠÙĦÙĦ غا +ĠÙĦÙĦغا ÙĬØ© +Ùħ ÙĪاÙĤ +ÙħÙĪاÙĤ Ùģ +ĠgÅĤówn a +Ġart Ä±ÅŁ +Ġ×ŀק ×ķ×ŀ×Ļ +ãĤ¯ãĥ© ãĥĸ +Ġس ÙĪÙī +ĠìŬ ìĦ± +اس ر +اسر ائÙĬÙĦ +Ġ׳ ×Ľ×ª×ij +ย à¹īà¸Ńà¸Ļ +Ġdeber á +Ġph ẫu +ÑİÑī ем +ĠÙĦدÙĬ ÙĨا +×ŀ×ĺ ×Ķ +Ġ׳ ×ķ׾×ĵ +ĠвÑģÑĤÑĢ еÑĩа +ãĤīãĤĮ ãģ¦ãģĦãģ¾ãģĻ +ĠcaÅĤ ej +ย ึ +ยึ à¸Ķ +поÑĤ ен +поÑĤен ÑĨи +Ġл иÑĤ +ĠлиÑĤ еÑĢ +ĠлиÑĤеÑĢ аÑĤÑĥÑĢ +Ġкажд ом +ĠíĮ IJ +ĠíĮIJ ëĭ¨ +à¸Ī ู +Ġpres ença +ãģªãĤĵ ãģ§ +Ùħ ÙĬاÙĩ +ин ÑĦоÑĢм +инÑĦоÑĢм аÑĨион +инÑĦоÑĢмаÑĨион н +ĠìŀIJ ìĹ° +ר׼ ש +Ġöd ül +ç¶ļ ãģı +Ġп Ñģ +ĠпÑģ иÑħ +ĠпÑģиÑħ олог +ت Ø°Ùĥر +Ġìŀħ ìŀ¥ +ล à¸Ķà¹Į +ìĦł ê±° +ãģ£ãģ¦ ãģĬãĤĬãģ¾ãģĻ +Ġ×Ļ ×¢ +Ġ×Ļ×¢ ק×ij +ĠاÙĦØ· عاÙħ +ãĥĨ ãĤ¹ãĥĪ +ĠTu ấn +Ġparticip ación +×ŀ×ķ×ŀ ×Ĺ×Ķ +×Ĵר ס×Ķ +ĠاÙĦتÙĨ ÙģÙĬ +ĠاÙĦتÙĨÙģÙĬ Ø°ÙĬ +ĠбезопаÑģ н +ge f +gef ähr +Ø´ ÙĪر +Ġmy ÅĽli +ÙĪا Ø´ÙĨ +ÙĪاشÙĨ Ø·ÙĨ +׳×ķס ×¢ +Ùĥ Ùĩ +ÙĥÙĩ رب +ÙĥÙĩرب اء +Ġmus iaÅĤ +ìĭ ¸ +ãĥĸãĥ© ãĥĥãĤ¯ +Ġcré é +ÙĨÙĩ ار +owo ÅĽÄĩ +ÙħØŃا ÙĥÙħ +ĠwÅĤa ÅĽ +ĠwÅĤaÅĽ c +ĠwÅĤaÅĽc iciel +ĠÙĬ ؤ +ĠÙĬؤ دÙĬ +×ŀ×¢ ×ķ׳ +×IJ ×ij׾ +خط Ø£ +ĠÑħ олод +×ĸ ×ķ׾ +ãģĵãĤĮ ãĤī +ãģĵãĤĮãĤī ãģ® +Ġbás ica +ฤ à¸Ķ +ฤà¸Ķ ูà¸ģ +ฤà¸Ķูà¸ģ า +ฤà¸Ķูà¸ģา ล +èIJ½ãģ¡ çĿĢ +ãģªãģĦ ãģĵãģ¨ +ص ÙĪÙħ +ÙĨج ØŃ +׳ק ×ķ×ĵ +׳ק×ķ×ĵ ת +кл аÑģÑģ +íķĺìĭľ ëĬĶ +ëĦ ĺ +Ġש×IJ ×Ļ׳×ķ +ĠС ейÑĩаÑģ +may acaģı +Ġyap ılır +Ġcategor ÃŃa +عب اد +ĠТ еп +ĠТеп еÑĢÑĮ +×Ķ×Ļס×ĺ ×ķר×Ļ +h ế +ãĤ³ ãĥ¼ãĥī +Ġcabe ça +ج Ùħا +جÙħا Ùĩ +جÙħاÙĩ ÙĬر +ä½İ ãģĦ +ĠÑĤоваÑĢ ов +à¸Ĭาว à¸ļà¹īาà¸Ļ +ĠÑģÑĤан ов +ĠÑģÑĤанов иÑĤÑģÑı +ĠавÑĤом обилÑĮ +ĠÑģлÑĥÑĩ ай +à¸Ńั à¸ŀ +ĠG iriÅŁ +ĠìĿ¼ ëĭ¨ +ĠпÑĢ оÑģ +ĠпÑĢоÑģ моÑĤÑĢ +ãģªãģıãģª ãģ£ãģŁ +มี à¸Ľà¸±à¸įหา +ïº İ +éc oute +ĠÙħ ÙĪجÙĪد +Ġس رÙĬع +ĠÙĪÙĩ ÙĨا +ĠÙĪÙĩÙĨا Ùĥ +à¸Ħุà¸ĵ สม +à¸Ħุà¸ĵสม à¸ļัà¸ķิ +Ġìļ° ìĦł +à¸ŀระ à¸ŀุà¸Ĺà¸ĺ +好 ãģ¿ +ظ ÙĦÙħ +Ġм акÑģ +ĠмакÑģ ималÑĮ +ĠмакÑģималÑĮ но +ãĥª ãĤ¢ãĥ« +à¹ģมà¹ī วà¹Īา +ĠاÙĦØŃ ÙĪار +ãĥĹãĥ© ãĤ¹ +Ġع ÙĦاÙĤØ© +Ġíĸī ëıĻ +Ġgönder il +Ġl ãi +ĠsaÄŁ lıkl +ĠsaÄŁlıkl ı +ĠÑĪ аг +Ġ×ij×IJר ×Ķ +prowadzi Äĩ +ãģĦãģı ãģ¤ãģĭ +Ġبت ارÙĬØ® +Ġ×ij×IJ×ķת ×Ķ +Ġmó c +ĠÐľ не +ãĥĹãĥ¬ ãĥ¼ +×IJ ×ĸר×Ĺ +åł´åIJĪ ãģ«ãģ¯ +使 ãģĪ +à¹Ģร ืà¸Ńà¸Ļ +ĠÐŁ еÑĤ +ĠÐŁÐµÑĤ ÑĢ +ãģ«åħ¥ ãĤĭ +Ùħ ادة +à¹Ģà¸ĩ ืà¹Īà¸Ńà¸Ļ +à¹Ģà¸ĩืà¹Īà¸Ńà¸Ļ à¹Ħà¸Ĥ +ĠÑģоÑģÑĤоÑı ние +ôn ica +ĠÑĦ ев +ĠÑĦев ÑĢа +ĠÑĦевÑĢа лÑı +Ġ×ķ ×ĸ +Ġ×ķ×ĸ ×IJת +à¸Ħร ิ +à¸Ħริ ส +ĠÐķ Ñīе +ãģ£ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĹãģŁ +ĠпÑĢав иÑĤелÑĮ +ĠпÑĢавиÑĤелÑĮ ÑģÑĤв +Ġtä glich +Ġëĭ¹ ìĭľ +×ŀ×ķ×¢ ×ŀ×ĵ +Ġдв оÑĢ +æī ķ +æīķ ãģĦ +ĠÑģÑĤан еÑĤ +Ġвозд ейÑģÑĤв +ĠвоздейÑģÑĤв и +Ġf ête +à¹Ģส า +תק ×ķ×ķ×Ķ +Ġu yar +Ġuyar ı +à¸ģลัà¸ļ à¹Ħà¸Ľ +Ġgi Æ°á»Ŀng +Ġв а +Ġва ÑĪи +ĠÄij áºŃu +ĠSpa ÃŁ +ĠìķĦ ë§Ī +à¹Ħà¸Ķà¹ī à¸ĩà¹Īาย +Ġ×Ķ×ŀ ×ijקש +æĸ° ãģŁ +æĸ°ãģŁ ãģª +ılı yor +пл ан +Ġ×Ķ×ijר ×Ļ×IJ×ķת +ĠaÄŁ rı +Ġsay gı +建 ãģ¦ +Ġnaj wyż +Ġnajwyż sz +سÙĬاس ات +ãģĬ å¾Ĺ +ĠاÙĦع ÙĦÙĬ +ĠاÙĦعÙĦÙĬ ا +Ġcoraz ón +ì¹ĺ ë£Į +หัว à¸Ĥà¹īà¸Ń +Ġب ØŃÙĬ +ĠبØŃÙĬ Ø« +зв езд +بÙĪ ابة +ÐĽ Ðĺ +ÙĦا زÙħ +Ġroz p +Ġrozp oc +Ġrozpoc zÄĻ +触 ãĤĮ +ĠاÙĦج ÙħÙĩ +ĠاÙĦجÙħÙĩ ÙĪر +Ġsp ÄĻd +ĠspÄĻd z +วิà¸Ĺยา ศาสà¸ķรà¹Į +ив аеÑĤÑģÑı +Ġдан ной +Ġreprés ente +ĠÄij á»ĭch +Ġ×¢×ŀ ×ķק +à¸Ńัà¸Ļ à¸ķร +à¸Ńัà¸Ļà¸ķร าย +Ġestr atég +Ġestratég ia +pad ÅĤ +Ġв полн +Ġвполн е +ĠпÑĢедоÑģÑĤав лен +×Ĺ׾ ×ķק +×Ĺ׾×ķק ת +ãĤ¢ ãĥĬ +ĠاÙĦغ Ø° +ĠاÙĦغذ ائÙĬ +ĠÑĥ зн +ĠÑĥзн аÑĤÑĮ +à¸ĭ à¹īาย +å½ĵ ãģ¦ +ØŃÙĬ اء +Ġbás ico +ק×ķ×ij ×¢ +ĠاÙĦÙħ باراة +ĠاÙĦÙĩ اتÙģ +Ġ׼ ׳×Ĵ×ĵ +à¸Ľà¸£à¸° หย +à¸Ľà¸£à¸°à¸«à¸¢ ัà¸Ķ +Ðļ ак +à¸Ĺีà¹Ī à¸Ļà¹Īา +à¸Ĺีà¹Īà¸Ļà¹Īา สà¸Ļà¹ĥà¸Ī +ãģ¾ ãģģ +ï½ ¢ +Ñģк оп +Ġson rasında +Ġur zÄħd +ĠurzÄħd zenia +׼×ķ ×ķ׳ +׼×ķ×ķ׳ ת +Ġ׾×Ķת ×ŀ×ķ×ĵ +Ġ׾×Ķת×ŀ×ķ×ĵ ×ĵ +ĠÑģ ли +ĠÑģли ÑĪ +ĠÑģлиÑĪ ком +ĠÑģÑĤ Ñĥд +ĠÑģÑĤÑĥд енÑĤ +Ġ×Ķ ×ķ×ĵ +Ġ×Ķ×ķ×ĵ ×¢×Ķ +ë¹Ħ ìļ© +à¸Ńยาà¸ģ à¹ĥหà¹ī +Ġb á»ģ +ยุ à¸Ĺà¸ĺ +Ðĺ ÐĿ +س ائر +Ø£ صÙĪÙĦ +ĠاÙĦغ رÙģ +ãģĵãģ¨ãĤĤ ãģĤãĤĬãģ¾ãģĻ +è¾¼ ãģ¾ãĤĮ +ĠاÙĦساب ع +Ġc ủ +ãģĦãģŁãģł ãģĦãģŁ +ì§ ĵ +ìĤ¬ 무 +powied ź +تÙģ Ùĥ +تÙģÙĥ ÙĬر +иÑĢов ки +ĠíĨµ íķ´ìĦľ +ãĤ¨ ãĤ¹ãĥĨ +ĠдеÑıÑĤелÑĮ ноÑģÑĤÑĮ +ĠданнÑĭ м +Ġ×¢ ×ķר +Ġ×¢×ķר ׼×Ļ +×ķ×ĵ עת +Ġhayat ını +Ġb Äħd +ĠbÄħd ź +obs ÅĤug +à¹Ģà¸ŀียà¸ĩ à¹ģà¸Ħà¹Ī +à¸ĭ à¹Īา +è²ł ãģij +ĠÑģÑĤÑĢ ем +ĠÄij á»īnh +ĠÐł ÑĥÑģ +ĠN ữ +Ġ׾×Ķש ×Ļ×Ĵ +Ġjed noc +Ġjednoc ze +Ġjednocze ÅĽnie +Ġ×Ķ×Ĵ ×ij×ķ×Ķ +أخ ÙĦاÙĤ +ĠнаÑģ ел +ĠнаÑģел ениÑı +ĠÙĬ ÙĨب +ĠÙĬÙĨب غÙĬ +ãģĮ ãģĭ +ãģĮãģĭ ãģĭ +×Ĵ עת +Ðŀ Ðł +ĠналиÑĩ ии +Ġë§Ī ì§Ģ +Ġë§Īì§Ģ ë§ī +Ġíĸī ìĤ¬ +Ġtre ÅĽci +Ġê°Ģ ì¹ĺ +ì¦ ĺ +Ġана лог +×Ķצע ת +в лад +влад е +ĠÑģдел ал +Ġ׳ ×Ĵ×Ļש +Ġ׳×Ĵ×Ļש ×ķת +полн ение +à¸Ĩ à¹Īา +ĠD ön +׼׾׼ ׾×Ķ +×ŀ×ĸ ×Ĵ +Ùħ Ùģ +ÙħÙģ Ùĩ +ÙħÙģÙĩ ÙĪÙħ +×Ķ ×ĵ +×Ķ×ĵ פס +×Ķ×ĵפס ×Ķ +ãģĻãģİ ãģ¦ +Ġг ÑĢ +ĠгÑĢ н +×ŀ×ĺ ×ķס +Ġ기 ìĸµ +ï¾ Ł +ĠpÅĤ yn +ĠGr ünde +ĠBü cher +Ġwed ÅĤug +ãģ¾ãģł ãģ¾ãģł +Ġ׳×Ķ ×ĵר +ĠÙĬست Ø·ÙĬع +ĠHi á»ĩp +ãĤŃãĥ£ãĥ³ ãĥļ +ãĤŃãĥ£ãĥ³ãĥļ ãĥ¼ãĥ³ +Ġth á»ķ +Ġeuropé enne +à¸ļ ัà¸ĩ +à¸ļัà¸ĩ à¸Ħัà¸ļ +ĠszczegóÅĤ owo +׳ שק +ãĥķ ãĥ©ãĥ³ãĤ¹ +×ŀ×ķ×ŀ ×Ĺ×Ļ +Ġcom ún +Ġç arp +ØŃت ÙĬا +ØŃتÙĬا ج +ØŃتÙĬاج ات +ëĭ´ ëĭ¹ +ä½ķ 度 +ä½ķ度 ãĤĤ +×ĵ ×ijק +ãģį ãĤĮ +ãģįãĤĮ ãģĦ +Ġк ам +Ġкам еÑĢ +ĠespecÃŃf ico +Ġtel éfono +à¸ķัà¹īà¸ĩ à¸Ńยูà¹Ī +I Åŀ +ãģ© ãĤĵãģ© +ãģ©ãĤĵãģ© ãĤĵ +עצ ×ŀ×IJ×Ļ +à¸Ķัà¸ĩ à¸Ļีà¹ī +ĠÑĦоÑĢм иÑĢов +ĠÑĦоÑĢмиÑĢов а +×ķ×ŀ ×ij +Ġkullan ımı +Ðľ Ðŀ +×¢ ש×Ļ +עש×Ļ ×Ļ×Ķ +Ġön lem +à¹Ģà¸Ń à¹ĩ +à¹Ģà¸Ńà¹ĩ ม +×ŀשק ×Ļ×¢ +ר ×Ļ×Ĺ +à¸Ĥ ัà¸Ķ +ĠíĻ ľ +ĠíĻľ ìļ© +à¸ĭ ะ +ãĤĪãģĨ ãģ«ãģªãĤĬãģ¾ãģĹãģŁ +ĠÑĢаÑģ пÑĢ +ĠÑĢаÑģпÑĢ оÑģÑĤ +ĠÑĢаÑģпÑĢоÑģÑĤ ÑĢан +ĠÑĢаÑģпÑĢоÑģÑĤÑĢан ен +׼×Ļ ×ķף +ÙĤب ض +تص رÙĬØŃ +تصرÙĬØŃ Ø§Øª +Ġо ÑĢи +ĠоÑĢи г +ĠоÑĢиг ина +ĠоÑĢигина л +ĠاÙĦع اÙĦÙĬ +à¹ģหà¹Īà¸ĩ à¸Ļีà¹ī +ãĥķãĤ¡ ãĥ¼ +ãģ¦ãģĦ ãģį +ãģ¦ãģĦãģį ãģŁãģĦ +פ תר +פתר ×ķ׳×ķת +Ġ×ij ×Ļ×Ĺ +Ġ×ij×Ļ×Ĺ ×ĵ +Ġod by +Ġodby ÅĤ +ĠоÑĩеÑĢ ед +Ġtr Æ°Æ¡ng +ãĤŃ ãĥ³ +×ŀ ×ķפ +×ŀ×ķפ ×¢ +ëĵľ 립 +ëĵľë¦½ ëĭĪëĭ¤ +à¸ŀืà¹īà¸Ļ à¸IJาà¸Ļ +ìŀIJ 격 +ĠVi á»ĩn +ĠDes pués +Ġ×IJ׾ ×Ļ׳×ķ +Ġdur ée +íĩ ´ +Ġmü zik +i ếu +ĠÑĢаз меÑīен +Ġк Ñĥд +ĠкÑĥд а +غ ض +غض ب +ĠTamb ém +à¸Īัà¸Ķ สà¹Īà¸ĩ +à¸ģาร à¹ģสà¸Ķà¸ĩ +onom ÃŃa +Ġан г +Ġанг ли +Ġангли й +Ġанглий Ñģк +Ġzn al +Ġznal az +Ġznalaz ÅĤ +תר ×Ĵ +תר×Ĵ ×ķ×Ŀ +ĠÑģ нов +ĠÑģнов а +ĠÑĩаÑģ а +Ġcommun auté +ĠespecÃŃf ica +ĠL á»ĭch +Ġli é +Ùģ جر +à¹Ģà¸ģ à¹Īà¸ĩ +ع اÙĦ +عاÙĦ ج +Ø£ÙĨ ظ +Ø£ÙĨظ ÙħØ© +ES Ä° +ĠاÙĦØŃ Ø¯ÙĬد +à¸ŀระ à¸Ńà¸ĩà¸Ħà¹Į +Ġפר שת +Ġдв иж +Ġдвиж ениÑı +ĠاÙĦج ارÙĬ +à¸ĺาà¸Ļ ี +неÑģ ен +ĠاÙĦÙĨ ÙĩائÙĬ +Ġб еÑĢ +ĠбеÑĢ ем +ĠбеÑĢем енн +Ġdépart ement +à¹Ģà¸Ĺ ีย +à¹Ģà¸Ĺีย à¸ļ +ĠÐľ аÑĢи +ĠнекоÑĤоÑĢ ÑĭÑħ +об еÑģп +обеÑģп еÑĩен +×Ĺ ×ķ×ĸ +×Ĺ×ķ×ĸ ×Ķ +ÙĨت ج +à¸Īะ à¹Ħà¸Ķà¹īรัà¸ļ +á» ° +Ġél éments +ع Ø· +عط اء +Ġt ắt +i á»ĩm +ÑİÑīиÑħ ÑģÑı +ãģĹãģ ° +ãģĹãģ° ãĤīãģı +Ġпом ожеÑĤ +à¸Ĥà¸ĵะ à¸Ļีà¹ī +Ġ×¢ שר×ķת +éģķ ãģ£ãģ¦ +ĠпÑĢ ог +ĠпÑĢог н +ĠпÑĢогн оз +Ġt ÅĤ +ĠtÅĤ um +ĠtÅĤum acz +T ür +Tür kiye +ãģį ãģ£ +ãģįãģ£ ãģĭãģij +Ġ×Ķ׳ ×ķ׼ +Ġ×Ķ׳×ķ׼ ×Ĺ×Ļ +ĠìĥĿ ìĤ° +ĠÑĦоÑĢм Ñĭ +ç¾İ ãģĹãģĦ +à¸Ľà¸£ ึà¸ģ +à¸Ľà¸£à¸¶à¸ģ ษา +Ġlum ière +ãĤª ãĥ¼ãĥĹ +ãĤªãĥ¼ãĥĹ ãĥ³ +à¸Ľ ืà¸Ļ +วั สà¸Ķ +วัสà¸Ķ ุ +еÑĢÑĤ в +ÙĥÙĦ Ùģ +ï½ £ +à¸ĺรรม à¸Ķา +׳ ×ĺר +ĠпÑĢедÑģÑĤав лÑıеÑĤ +Ġanál isis +Ġb ãi +با ÙĤÙĬ +à¸Ľà¸£à¸° à¹Ģà¸Ķ +à¸Ľà¸£à¸°à¹Ģà¸Ķ à¹ĩà¸Ļ +ĠÑģлÑĥÑĩ аÑı +ĠÑģлÑĥÑĩаÑı Ñħ +ÐĽ ÐIJ +สัà¸ĩ à¹Ģà¸ģ +สัà¸ĩà¹Ģà¸ģ à¸ķ +Ġprz ec +Ġprzec ież +Ùħ صÙĦ +ÙħصÙĦ ØŃØ© +ש×ķק ×ķ׾×ĵ +ĠобоÑĢÑĥд ованиÑı +Ġtr waÅĤ +رÙĪ Ùħ +ìķĪ ëĤ´ +ĠNgh á»ĭ +Ø® Ø´ +à¸ļา à¸Ħาร +à¸ļาà¸Ħาร à¹Īา +Ġоп ÑĨион +ĠÑģозд аниÑı +ãĤ³ ãĤ¹ãĥĪ +Ġ×Ķ×¢ ׾×Ļ +Ġ×Ķ×¢×ľ×Ļ ×ķף +lä uft +ãĥĻ ãĤ¹ãĥĪ +Ġr ê +Ġrê ve +×IJ ×ij×Ļ×ij +×Ļ ×Ļ×ļ +ë¶ Ļ +ãĤ¤ãĥ³ ãĥī +ÅĤo ży +ÅĤoży Äĩ +ع ائÙĦ +عائÙĦ Ø© +Ø£ ÙĪر +Ø£ÙĪر اÙĤ +à¸Ĺà¹īà¸Ńà¸ĩ à¸ĸ +à¸Ĺà¹īà¸Ńà¸ĩà¸ĸ ิà¹Īà¸Ļ +Ġä hn +Ġähn lich +ãĥŁ ãĥĭ +à¸ľ ู +à¸ľà¸¹ à¹īà¸Ļ +à¸ľà¸¹à¹īà¸Ļ ำ +ĠмаÑĤеÑĢиал Ñĭ +Ġкап иÑĤ +ĠкапиÑĤ ал +ï¼ ¦ +Ġseç il +Ġh ứng +Ġintéress ant +ãģ£ãģ¦ ãģĦãģı +Ġe ÄŁer +ëIJĺ ìĹĪìĬµëĭĪëĭ¤ +Ġan laÅŁma +ãģĶ åĪ©çĶ¨ +Ġ×ij ×ĸ׼ +Ġ×ij×ĸ׼ ×ķת +ëĿ¼ ë©´ +ĠÙĬ ÙĪس +ĠÙĬÙĪس Ùģ +أسÙĦ ØŃØ© +ĠGef ühl +ĠноÑĢм алÑĮн +ãĥĻ ãĥ³ +ãģķãĤĮ ãĤĭãģĵãģ¨ +ĠÐij еÑģ +ãģ¨ãģĦ ãģĪãģ° +ĠÙħ ÙĩÙħ +ĠÙħÙĩÙħ Ø© +ãģ§ãģĹãĤĩãģĨ ãģŃ +ĠêµŃ ëĤ´ +à¹Ģม à¹ĩà¸Ķ +×ŀ×ij קר +ĠاÙĦد ÙĨÙĬ +ĠاÙĦدÙĨÙĬ ا +à¸Ĭ ู +к ÑĢÑĥÑĤ +Ġtho áng +Ġ׳ ×ĵר +Ġ׳×ĵר ש +ĠÑĢаÑģÑģ казал +ĠAu ÃŁerdem +פ ×IJר +פ×IJר ק +Ġ×ŀש×Ĺק ×Ļ×Ŀ +צ ר׼×Ļ×Ŀ +×ŀ×ĵ ×ķ +×ŀ×ĵ×ķ ×Ļק +èĭ¦ ãģĹ +ĠÑģ иг +ĠÑģиг нал +ĠM á»įi +Ġtr ữ +Ġnast ÄĻp +ĠnastÄĻp nie +Ġì¶Ķ ì§Ħ +ĠاÙĦÙģ ÙĨد +ĠاÙĦÙģÙĨد ÙĤ +koÅĦ czyÅĤ +ส ีà¹Ī +ק ×Ļ×ij +ק×Ļ×ij ×ķ×¥ +ĠнÑĥж нÑĭ +大 åĪĩ +大åĪĩ ãģª +æıĽ ãģĪ +ת ×ķס +ת×ķס פת +ãģ£ãģ¦ ãģĦãģªãģĦ +Ġм Ñı +ĠмÑı г +ĠмÑıг к +Ġjak ie +Ġjakie ÅĽ +à¸ķำ à¸ļ +à¸ķำà¸ļ ล +ĠìŀĪ ì§Ģ +×ij×ĺ ×IJ +ĠоÑĤлиÑĩ но +ÙĤ ÙIJ +ĠавÑĤом об +ĠавÑĤомоб и +ĠавÑĤомоби лÑı +دÙĬÙħÙĤرا Ø·ÙĬ +ĠاÙĦ ÙĪا +ĠاÙĦÙĪا ØŃد +Ġس ÙĪرÙĬØ© +Ø£ غÙĦ +أغÙĦ ب +ĠÑįк ÑĢан +ãĥĹ ãĥ©ãĤ¤ +Ġjeste ÅĽ +ãĥIJ ãĥª +Ġ×Ķ×IJ ×ķ×ķ×Ļר +ائ Ùĥ +à¸Ńยà¹Īาà¸ĩ ยิà¹Īà¸ĩ +ÑĢ екÑĤ +Ġum o +Ġumo ż +Ġumoż li +Ġumożli w +Ġumożliw ia +Ġnäch ste +ĠìŀĪ ì§Ģë§Į +ĠпÑĢед н +ĠпÑĢедн аз +ĠпÑĢедназ наÑĩен +Ġma çı +Ġp omi +Ġpomi ÄĻd +ĠpomiÄĻd zy +ĠاÙĦÙĦ ÙĤاء +à¹Ģà¸Ķ à¸Ńะ +Ġнов оÑģÑĤи +×ŀ׊׾×Ķ +رÙĬاض ÙĬ +à¸Ķ à¸Ļ +à¸Ķà¸Ļ à¸ķรี +ب صر +ìĬ¤ íĥĢ +scri pción +Ġnap isa +Ġnapisa ÅĤ +Ġ׳ש ×ŀ×¢ +ĠاÙĦÙħØŃ ÙĦÙĬ +Ġhi á»ĥn +×IJ ×Ĺ +×IJ׊ר×IJ×Ļ +Ġг ÑĢаниÑĨ +æīĭ ç¶ļãģį +Ùĥ سب +Ġà¹ģà¸ķà¹Ī à¸ĸà¹īา +à¸Ķาว à¸Ļà¹Į +à¸Ķาวà¸Ļà¹Į à¹Ĥหลà¸Ķ +ãĤĭãģĵãģ¨ãģĮãģ§ãģį ãģ¾ãģĻ +åŁºæľ¬ çļĦãģ« +ÙĪÙĦ اد +rä ume +د Ùģاع +×Ļצ ×¢ +ĠO czy +ĠOczy wiÅĽcie +ĠÅ ģ +ĠÅģ a +اÙĦÙĬ اب +اÙĦÙĬاب اÙĨ +áºł I +ĠBir liÄŁi +×Ķ ×ķצ +×Ķ×ķצ ×IJת +ĠÄij ua +Ġê·¸ëŁ¬ ëĭĪê¹Į +Ġréal ité +ع ÙĦاÙĤات +J este +Jeste ÅĽ +Ġмн ож +Ġмнож еÑģÑĤво +ï¼ « +ãĥĹãĥŃ ãĤ¸ãĤ§ +ãĥĹãĥŃãĤ¸ãĤ§ ãĤ¯ãĥĪ +ĠÑĦ л +ظ ÙĨ +×Ĵ׾ ×Ĵ׾ +ĠmÅĤod zie +ĠmÅĤodzie ż +à¸Ļà¹īำ à¸ķา +à¸Ļà¹īำà¸ķา ล +ÐĽ Ðķ +×ij ×ķ×ĺ +Ġ׾×Ķ ×Ĵ×Ļ×ĵ +ãģĵãģ¨ãĤĤ ãģĤãĤĭ +ز اد +×ŀ×Ļ×ĵ ×¢ +ĠgÅĤówn ie +ãĥı ãĤ¦ +ãĥıãĤ¦ ãĤ¹ +б ел +Ġét ape +ðŁĺ Ģ +Ġмод елÑĮ +a ģını +ש ×Ĺק +ש×Ĺק ף +Ġni ño +à¸Ĭ à¹īาà¸ĩ +à¹Ģล ีย +ĠÑĦоÑĢм е +ĠاÙĦØ´ رÙĬÙģ +ĠÑĥд аÑĢ +arr iv +arriv ée +Ġmies iÄĻ +ĠmiesiÄĻ cy +ØŃ Ø±Ùĥ +ØŃرÙĥ ات +ĠDi á»ħn +ÐĿ Ы +ãģ¾ãģ£ãģŁ ãģı +Ġ×Ļ ר×ķק +еÑģÑĤ еÑģÑĤв +еÑģÑĤеÑģÑĤв енн +Ġê·¸ ëŁ¼ +ĠاÙĦÙħ تÙĪ +ĠاÙĦÙħتÙĪ سط +Ġbéné fic +Ġbénéfic ie +Ġwy bra +Ġwybra Äĩ +ĠاÙĦز ÙħÙĨ +ĠпÑĢин Ñı +ĠпÑĢинÑı л +Ùģر ØŃ +Ġk sz +Ġksz taÅĤ +ĠksztaÅĤ t +ק׾ ×ĺ +×ij×ĵ×Ļק ת +Ġgi ấ +Ġgiấ c +Ġpropriet Ãł +деÑĢж ан +ĠKö ln +ĠGü zel +×Ļפ ×ķ×Ļ +ĠCu á»Ļc +ÑįÑĤ аж +تر ÙĥÙĬ +ترÙĥÙĬ ز +лож ений +Ġп Ñĥ +ĠпÑĥ ÑĤи +اخت ÙĦاÙģ +åĩºãģ¦ ãģıãĤĭ +à¸ļุ à¸ģ +âĿ ¤ +ÑĦ ан +פש ×ĺ +à¸ļัà¸Ļ à¹Ģà¸Ĺ +à¸ļัà¸Ļà¹Ģà¸Ĺ ิà¸ĩ +ĠاÙĦس اد +ĠاÙĦساد س +ĠاÙĦÙĤ ÙĪÙħ +ĠاÙĦÙĤÙĪÙħ ÙĬ +Ġyönet ici +Ùĩ ÙĪات +ÙĩÙĪات Ùģ +Ġrespons ável +Ġпод деÑĢжива +ĠاÙĦسÙĦ Ø· +ĠاÙĦسÙĦØ· ات +ãģĹãģ¦ ãģĬãģı +ãĥļ ãĥĥãĥĪ +à¸Ľ ุà¹Īม +Ġogl Äħda +ÙĨا ÙĤ +ÙĨاÙĤ Ø´ +à¸Ħà¸Ńà¸Ļ à¹Ĥà¸Ķ +ĠMü sl +ĠMüsl ü +ĠMüslü man +ĠMo ż +ĠMoż na +Ġnum érique +Ġv á»ı +ĠسÙĬ تÙħ +Ġyer leÅŁ +монÑĤ аж +Ġgo ût +ãģ¦ ãģĬãĤĬãģ¾ãģĻ +ĠKh ánh +Ġе дин +Ġедин ÑģÑĤв +اÙĨ Ø®Ùģ +اÙĨØ®Ùģ اض +ìĭľ íĹĺ +Ġl ặng +ĠÑĢ олÑĮ +à¸ķัว à¹ģà¸Ĺà¸Ļ +à¸Ħà¹Īา à¹ĥà¸Ĭà¹ī +à¸Ħà¹Īาà¹ĥà¸Ĭà¹ī à¸Īà¹Īาย +Ġver füg +Ġverfüg bar +ìĻĶ ëĭ¤ +ãģĦ ãģļ +ãģĦãģļ ãĤĮ +ĠиÑģÑģлед ованиÑı +меÑī а +×Ķ ×Ĺ +×Ķ×Ĺ ×ĸר +à¹ģà¸Ł à¸Ĭัà¹Īà¸Ļ +ت صرÙģ +Ø¥ رÙĩاب +Ġexerc ÃŃcio +Ġé lev +Ġélev é +สัà¸įà¸įา à¸ĵ +Ãĸ Z +ãĥĹ ãĥŃãĤ° +ãĥĹãĥŃãĤ° ãĥ© +ãĥĹãĥŃãĤ°ãĥ© ãĥł +Ġw ewnÄĻtrzn +Ġhen üz +é£Ľ ãģ³ +à¹Ģà¸Ķ à¸Ńรà¹Į +Ñģ Ñĥж +ÑģÑĥж ден +شع ÙĪب +ãģ²ãģ¨ ãĤĬ +Ġwy ÅĤÄħ +ĠwyÅĤÄħ cznie +Ġпло Ñħо +ÐĶ Ðķ +Ạ¦ +Ùģع اÙĦÙĬ +ÙģعاÙĦÙĬ ات +ĠاÙĦع شر +ÑģÑĤÑĥп ил +Ġy arg +Ġyarg ı +нÑİ Ñİ +×ķ×IJ ×ij +Ġu ç +Ġuç ak +ë² ½ +تÙĪ ÙĤÙĬ +تÙĪÙĤÙĬ ع +Ġì¤ij ìĭ¬ +׳×Ļ×ķ ×ķ×ĺ +Ø£ ÙĥÙĦ +ç½® ãģĦãģ¦ +éłĤ ãģį +Ġ×Ķת ×ij +Ġ×Ķת×ij ×Ļ×¢×Ķ +Ġdür fen +Ùħ ÙĤاÙĦ +ÙħÙĤاÙĦ ات +Ġز ÙħÙĨ +à¸ŀฤ ศ +à¸ŀฤศ à¸Ī +à¸ŀฤศà¸Ī ิà¸ģ +à¸ŀฤศà¸Īิà¸ģ ายà¸Ļ +ĠнеÑģк олÑĮ +ĠнеÑģколÑĮ ки +ĠнеÑģколÑĮки Ñħ +Ġcrian ça +มิ à¸ķร +×ŀ׼ ×Ļר×ķת +à¸ģาร à¸ļริหาร +Ġtélé charg +Ġ×IJ×ķ×Ķ ×ijת +ĠBü ro +ä½ľ ãģ£ãģŁ +ĠKi ÅŁi +ç¾İåij³ ãģĹ +à¹Ģลย à¸Ħà¹Īะ +à¸ŀà¸ļ à¸ģัà¸ļ +à¸Ī à¹īา +Ġç er +Ġçer ç +Ġçerç eve +ãĤĴä½ľ ãģ£ãģ¦ +ĠпеÑĢв ÑĥÑİ +×ŀצ ר×Ļ×Ŀ +×IJ׾ ×ķ×Ķ +×IJ׾×ķ×Ķ ×Ļ×Ŀ +Ġagr é +Ġagré able +Ġay ır +Ä°L Ä° +ãĤ ¥ +Ġíĺ Ħ +ĠíĺĦ ìĭ¤ +ثاÙĦ Ø« +ת ×ĸ +ת×ĸ ×ķ׳×Ķ +ãģ¨ãģĦ ãģ£ãģ¦ +ãģ¨ãģĦãģ£ãģ¦ ãĤĤ +Ġا بÙĪ +ĠÑģоб ак +é£Łãģ¹ ãģŁ +Ġдан ном +à¹Ģล ิ +à¹Ģลิ ศ +Ġí ļ +Ġíļ ¨ +Ġíļ¨ ê³¼ +ãĤĤãĤī ãģĪãĤĭ +׳ צ׾ +ÑĦ ик +ÑĦик Ñģ +Ġjeste ÅĽmy +ת×Ĺ×ķש ×Ķ +à¹Ħมà¹Ī à¸Ħวร +ĠØŃ Ø³ÙĬÙĨ +à¸ģาร ลà¸ĩà¸Ĺุà¸Ļ +ë´ ¤ +ĠÐĺ менно +à¸ļ à¸Ńรà¹Į +à¸ļà¸Ńรà¹Į à¸Ķ +ĠC ảnh +ìĦľ ë¹ĦìĬ¤ +Ġпол ов +Ġполов ин +Ġзам еÑĩа +ãģĦãĤį ãĤĵãģª +Ġ×ij ×Ļק +Ġ×ij×Ļק ש +л ÑĥÑĪ +ãĤĴ è¿İ +ãĤĴè¿İ ãģĪ +جرÙĬ ÙħØ© +Ġt ây +ĠاÙĦÙĨ ÙĪ +ĠاÙĦÙĨÙĪ ÙĪÙĬ +ÃĤ N +ì¿ ł +หà¸Ļ าว +Ġ×ij׊ש×ij×ķף +ز ار +à¸Ķ าร +à¸Ķาร า +ĠÅĽ l +ĠÅĽl ub +มีà¸Ħวาม สุà¸Ĥ +Ġn hu +Ġnhu áºŃn +ÙħØŃ Ø·Ø© +à¹Ģสืà¹īà¸Ń à¸ľà¹īา +ĠТ олÑĮко +ĠÙĥ س +ĠÙĥس ارة +ÙħØ´ رÙĪع +niÄĻ cia +×¢ ׼ש×Ļ×ķ +ت ÙĦÙģ +تÙĦÙģ زÙĬ +تÙĦÙģزÙĬ ÙĪÙĨ +Ġl Æ°á»Ľi +ĠÐľÐ¾Ñģк вÑĭ +Ġré serve +Ġan laÅŁ +ĠanlaÅŁ ıl +Ġed eceÄŁi +รà¸Ńà¸ĩ à¹Ģà¸Ĺà¹īา +Ġب Ø· +Ġبط رÙĬ +ĠبطرÙĬ ÙĤØ© +ãģ¦ãģĹãģ¾ ãģ£ãģ¦ +ãĤĤãĤī ãģ£ãģ¦ +بر ج +æ± ļ +æ±ļ ãĤĮ +Ġch oc +Ġchoc ia +Ġchocia ż +Ġzob ac +Ġzobac zyÄĩ +пÑĢ Ñı +пÑĢÑı жен +ĠÑĨ иÑĦ +ĠÑĨиÑĦ ÑĢ +Ġм ам +Ġвз ÑıÑĤÑĮ +Ġch ạm +ج سÙħ +ØŃÙħ اس +à¹Ģล à¹Īม +à¸ŀิ ษ +×Ķפ ׼×ķ +à¸Ĭà¹Īà¸Ńà¸ĩ à¸Ĺาà¸ĩ +Ġв ек +Ġвек а +Æ¡ Ìģ +Æ¡Ìģ i +ĠTi á»ģn +Ġtr ầm +мÑĭ ÑĪ +мÑĭÑĪ л +ĠÑĤ Ñĥ +ĠÑĤÑĥ ÑĢиÑģÑĤ +Ġch c +Ġchc Äħ +Ġав г +Ġавг ÑĥÑģÑĤ +ĠавгÑĥÑģÑĤ а +ס ×IJ×ķת +Ġר ×Ĵ׾ +à¸ľà¸¥ à¸ģระà¸Ĺ +à¸ľà¸¥à¸ģระà¸Ĺ à¸ļ +å¤īãĤı ãĤĭ +Ġ×Ķ×IJ×Ĺר ×ķ׳×Ļ×Ŀ +سÙģ ÙĬر +ĠÑĩа Ñīе +ãģĦ ãĤī +ãģĦãĤī ãģ£ +ãģĦãĤīãģ£ ãģĹãĤĥ +×ķ×ŀ ׳×Ļ×Ŀ +Ġart tır +ĠCh á»ĭ +Ġì¡° ì§ģ +ĠÑĥÑģп еÑħ +Ġ×¢ ×ķס +Ġ×¢×ķס ק +ĠìĥĿ ëªħ +ÑĨ иÑĤ +Ġreg ión +Ðŀ ÐĿ +ĠdoÄŁ um +ĠyaÅŁ ad +ĠyaÅŁad ıģı +à¸Ĺà¸Ķ ลà¸Ńà¸ĩ +Ġgöz ü +ש ×Ļר×Ķ +дÑĥм ал +Ġda ģı +Ġdaģı t +à¸Ĺีม à¸ĩาà¸Ļ +Ġti á»ģm +ĠاÙĦÙĥ بر +ĠاÙĦÙĥبر Ùī +ì¹ Ń +ĠGü nc +ĠGünc elle +ĠGüncelle me +ê¹ Ĭ +ĠобоÑĢÑĥд ование +ĠÑĢеÑĪ а +á» ¤ +Ġп иÑĤ +ĠпиÑĤ аниÑı +à¹Ģรีย à¸ļ +×Ľ×ª ×Ļ×ij×Ķ +Ġп он +Ġпон ÑĢав +ĠпонÑĢав и +Ġ×Ķ ×ķ׾×ĵ +Ġ×Ķ×ķ׾×ĵ ת +Ġê² ģ +Ġê²ģ ëĭĪëĭ¤ +ĠпеÑĢв ой +ãĥ©ãĤ¤ ãĥķ +ĠÅŁi ir +kr ÄĻ +krÄĻ c +Ġthi á»ĥu +à¹Ģลย à¸Ĺี +à¹Ģลยà¸Ĺี à¹Ģà¸Ķียว +×ĺ×¢ ׳×ķת +ائ ÙĩÙħ +Ġ×IJ ס×ķר +ĠплаÑĤ еж +تر دد +Ġmożli we +Ġkh Ỽ +ĠkhỼ p +تÙģاع ÙĦ +ĠÑĪ колÑĮ +ĠÑĪколÑĮ н +ĠÙĤ صة +Ġmét ier +nÄĻ ÅĤa +หล à¹Īà¸Ń +Ġ ủng +Ġprz egl +Ġprzegl Äħd +ĠاÙĦÙħ تعÙĦ +ĠاÙĦÙħتعÙĦ ÙĤØ© +ĠÑģÑĭ н +Ġв олн +ãĥĩ ãĥ¼ãĥĪ +ĠÐŃ ÑĤи +Ġк ÑĢоме +à¸Ħ ารà¹Į +׳ק ×ķ×ĵ×Ķ +Ġ׾ש×ŀ ×ķ×¢ +Ġ×ĸ ×ķ׼ר +ï¼ § +ÙĬ Ùİا +Ġgi á»ıi +åĥį ãģı +ĠÑģ ни +ĠÑģни жен +à¹ģà¸Ķ à¸Ķ +รุ à¸Ļ +รุà¸Ļ à¹ģรà¸ĩ +Ġhi á»ĩp +ograf ÃŃa +à¹Ģà¸Ī à¸Ńรà¹Į +Ġдв иг +Ġдвиг аÑĤ +ĠдвигаÑĤ ел +Ġü y +Ġüy eler +Ġüyeler i +Ġб Ñĥк +ĠбÑĥк в +ãĤĤ å¤ļãģı +Ġthi á»ĩt +ĠPa ÃŃs +ĠØ· بÙĬعÙĬ +à¹ģà¸Ī à¸ģ +ĠاÙĦص ØŃÙĬØŃ +Ġapp ré +Ġappré ci +Ġdecis ión +Ġë°ĺ ëĵľ +Ġë°ĺëĵľ ìĭľ +ĠÑĤеб е +ãĤ· ãĥ¼ãĤº +ãĤ·ãĥ¼ãĤº ãĥ³ +Ġд алÑĮн +ĠìĬ ¤ +ĠìĬ¤ ìĬ¤ +ĠìĬ¤ìĬ¤ ë¡ľ +ĠTh á»ĥ +Ġkar ÅŁ +ĠkarÅŁ ıs +ĠkarÅŁÄ±s ında +ĠK ön +ĠKön ig +ив ание +×ij ×ķצע +г лаÑģ +Ġtw ó +Ġtwó rc +à¸Ľà¸ģ à¸Ħร +à¸Ľà¸ģà¸Ħร à¸Ńà¸ĩ +ĠG ÅĤ +ĠGÅĤ ówn +ĠUnter stüt +ĠUnterstüt zung +Ġд ÑĥÑħ +ĠдÑĥÑħ ов +Ø£ ÙħاÙĨ +×Ĺש ש +ت ظ +تظ اÙĩر +ĠлÑİб ом +à¸ķ าร +à¸ķาร าà¸ĩ +Ġkr ól +Ø£ ØŃدث +ì¡Į ëĭ¤ +Ðļ ÑĥÑĢÑģ +ãĥĥ ãĥĦ +×ŀק ×ķ×ij׾ +ĠÑģимв ол +Ġdés orm +Ġdésorm ais +w üns +wüns che +Ñĥ ни +Ñĥни ÑĨип +ÑĥниÑĨип алÑĮн +หลัà¸ģ สูà¸ķร +ÙĨت شر +Ġа л +Ġал к +Ġалк ог +Ġалког ол +ĠÑĥ ÑĩиÑĤÑĭва +à¸ģำ à¸ģัà¸ļ +Ġ׾ פע×ķ׾ +ĠìĹ° ê²° +s Äħd +ĠاÙĦØ£ ÙĬ +ĠاÙĦØ£ÙĬ اÙħ +غÙĬ اب +Ġна ÑĢ +ĠнаÑĢ ко +×ŀ×ķ×ĵ ×¢ +ĠÑģеÑĢ ии +пиÑģ Ñĭва +สิ ว +ç¶ļ ãģĦãģ¦ +çĶ³ãģĹ è¾¼ãģ¿ +Ġ׾ ×Ĵר +Ġ׾×Ĵר ×ķ×Ŀ +Ġд ем +Ġдем о +Ġë³´ ëĤ´ +تÙĩ دÙĬد +ĠÙħØ´ ÙĬرا +Ġdu y +Ġduy á»ĩt +ĠwiÄĻks ze +Ùħع اÙĬ +ÙħعاÙĬ ÙĬر +ĠG da +ĠGda ÅĦsk +Ġr ah +Ġrah ats +Ġrahats ız +ר ×ķצ×Ķ +l ös +lös ung +ĠТак им +ÑĪ ед +ÑĪед ÑĪ +ع زÙĦ +Ġרש ×Ļ×ŀת +Ġ׾×Ķ ×Ļ׼ +Ġ׾×Ķ×Ļ׼ ×ł×¡ +Ġп ÑĥÑĤ +ĠпÑĥÑĤ еÑĪ +ĠпÑĥÑĤеÑĪ еÑģÑĤв +Ġnot ÃŃcia +Ġal Ä±ÅŁ +ĠalÄ±ÅŁ ver +ĠalÄ±ÅŁver iÅŁ +ĠwÅĤ os +ĠwÅĤos ów +Ġب غ +Ġبغ داد +Ġver öffent +Ġveröffent licht +ĠKh á +Ġt án +ëIJĺ 기 +Ġë°© 문 +Ùģ ÙĬÙĦ +à¹Ģà¸ģิà¸Ķ à¸Īาà¸ģ +åı¯ æĦĽ +åı¯æĦĽ ãģĦ +à¸ĸ ุà¸ĩ +Ġz ewnÄĻtrzn +à¸łà¸²à¸©à¸² à¸Ńัà¸ĩà¸ģฤษ +Ġmá xima +Ġul us +Ġulus lararası +Ġ׳×Ķ ׳ +à¸Ĥà¹Īาว สาร +ĠìĿĺ ìĤ¬ +à¹Ģหล ืà¸Ńà¸ĩ +Ġد ÙĤ +ĠدÙĤ ائÙĤ +สืà¹Īà¸Ń สาร +ë¨ ¼ +ĠÑģоÑģÑĤоÑı нии +สมา à¸Ħม +á» Ĥ +ĠÐľÐ¾Ñģ ков +ĠÐľÐ¾Ñģков Ñģк +×ŀס ×ķ×Ĵ׾ +ãģĭ ãģĭãĤĬ +ĠTr uyá»ģn +à¹ģà¸Ĥà¹ĩà¸ĩ à¹ģรà¸ĩ +×ŀ×Ĺ ×ĸ×Ļק +à¹Ĥà¸ģ à¹ī +ÙĬس ر +ìĶ © +×IJ ×ķק +×IJ×ķק ×ĺ +×IJ×ķק×ĺ ×ķ×ijר +Ġprox imité +ÙħÙĨ Ùĩج +ĠاÙĦج ز +ĠاÙĦجز ائ +ĠاÙĦجزائ رÙĬ +ĠÄIJi á»ĥm +Ġден еж +Ġденеж н +ÙģØŃ Øµ +Ùģ ئ +ĠÐij Ñĥд +×Ĵ×Ļ×ĵ ×ķ׾ +ĠÐĴ едÑĮ +عÙĦ اÙħØ© +Ġ×IJ×Ĺר ×ķ׳×ķת +ãģĦãģŁãģł ãģĦãģ¦ +سÙĦ ØŃ +ØŃ ÙĦÙħ +ز ÙĪار +Ùĥ سر +×ĺ קס +Ġб ан +Ġбан ков +ĠпÑĢ ож +ĠпÑĢож ива +li wo +liwo ÅĽci +ĠTi ếp +ĠاÙĦÙħÙĨ اسب +ĠاÙĦØ® ÙĬار +ãģĬ ãģĭ +ãģĬãģĭ ãģĴ +à¸Ķà¸Ńà¸ģ à¹Ħมà¹ī +ä mp +ämp fe +à¸ķัà¹īà¸ĩ à¹ĥà¸Ī +Ġза ÑīиÑĤ +ĠзаÑīиÑĤ Ñĭ +ĠTh Æ°á»Ŀng +Ġص Ùģ +ĠصÙģ ØŃØ© +×Ĺ×ķר ×£ +ãĥIJ ãĥĥãĤ° +Ġ×ĵ ×Ļ×Ĵ +Ġ×ĵ×Ļ×Ĵ ×Ļ×ĺ +Ġ×ĵ×Ļ×Ĵ×Ļ×ĺ ׾×Ļ +Ġ×Ķ×Ĺ ×ķ׾×Ļ×Ŀ +в еÑī +веÑī а +Ġк ÑĥлÑĮÑĤ +ĠкÑĥлÑĮÑĤ Ñĥ +ĠкÑĥлÑĮÑĤÑĥ ÑĢÑĭ +ĠاÙĦاÙĨ ترÙĨت +Ġhö ch +Ġhöch st +Ġíĺ ķ +Ġíĺķ íĥľ +Ġв ой +Ġвой нÑĭ +ÐĽ Ðŀ +ìĭł ìļ© +Ġ×ŀ×ij ×ķס +Ġ×ŀ×ij×ķס ס +×ŀ׳ ×Ļ×¢ +Ġfiyat ı +ĠÑģл Ñĥж +ĠÑģлÑĥж бÑĭ +à¸Ĺั ศ +à¸Ĺัศ à¸Ļ +ãģĵãģ¨ãģĮ å¤ļãģĦ +Ġ×Ķ×ŀש ת +Ġ×Ķ×ŀשת ×ŀש +å¯Ħ ãģĽ +×ŀש׾ ×ķ×Ĺ +æĻĤ çĤ¹ +æĻĤçĤ¹ ãģ§ +à¸ŀร ี +à¸ŀรี à¹Ģมีย +à¸ŀรีà¹Ģมีย รà¹Į +à¸ŀรีà¹Ģมียรà¹Į ลีà¸ģ +Ġdiffic olt +Ġdifficolt Ãł +ãĥ¬ ãĤ¹ãĥĪ +ãĥ¬ãĤ¹ãĥĪ ãĥ©ãĥ³ +สม à¹Ģà¸Ķà¹ĩ +สมà¹Ģà¸Ķà¹ĩ à¸Ī +Ġж ид +Ġжид к +Ġzu peÅĤ +ĠzupeÅĤ nie +ĠÙħ جر +ĠÙħجر د +ãģĮ å§ĭ +ãģĮå§ĭ ãģ¾ +ãĤŃãĥ£ ãĥ© +Ġ×IJ ×ķ×ķ×Ļר +ãģĬ äºĴ +ãģĬäºĴ ãģĦ +Ġpot rÃł +ĠPa ÅĦst +ĠPaÅĦst wo +Ġب ÙĬاÙĨ +ĠبÙĬاÙĨ ات +Ġин огда +ĠÑĢ а +ĠÑĢа ÑģÑĤв +ĠÑĢаÑģÑĤв оÑĢ +Ġ×ĸ ×ŀ׳ +ยิ à¹īม +Ä Ĩ +ãģ¾ ãģķ +ãģ¾ãģķ ãģ« +ãĥķãĤ¡ ãĤ¤ãĥ« +Ġgörd Ã¼ÄŁÃ¼ +สà¸ĩ à¸Ħร +สà¸ĩà¸Ħร าม +ĠArk adaÅŁ +ĠrozwiÄħz ania +×ŀ ×ķ×ĺ +pi ÄĻ +piÄĻ t +ص غر +ส ย +สย าม +ãĤĨ ãģ£ãģıãĤĬ +Ġtr ần +Ġeconom ÃŃa +Ġgeh ören +ãĤ·ãĥ§ ãĥ¼ +ĠsÅĤ ucha +à¸ŀà¸Ń à¹ĥà¸Ī +ĠоÑĤмеÑĤ ил +ÙĨت ÙĤÙĦ +Ġprop ósito +ĠваÑĪ его +Ġnh ắn +à¹ģà¸ĸ ว +Ġком иÑģ +ĠкомиÑģ Ñģи +waż nie +Ġy avaÅŁ +×ŀ ×Ļק +×ŀ×Ļק ×ķ×Ŀ +ש×IJ׾ ת +Ġyıll arda +ĠÐ ® +ĠЮ ÑĢ +×ł×¡ ×Ļ×ij×ķת +ת צ +תצ ×ķ×Ĵ +Ġод нÑĥ +Ġ à¸Ńยà¹Īาà¸ĩà¹Ħร +Ġà¸Ńยà¹Īาà¸ĩà¹Ħร à¸ģà¹ĩà¸ķาม +ëģ ¼ +à¹Ħล à¹Ī +تس ÙĦÙĬÙħ +بÙĦ اغ +Ġì ī +Ġìī ½ +Ġìī½ ê²Į +ãĥļ ãĥ³ +зв ÑĥÑĩ +ĠW äh +ĠWäh rend +Ġ×Ļ ×Ļת +Ġ×Ļ×Ļת ׼ף +Ġkh uyên +Ġv ẽ +Ġа меÑĢ +ĠамеÑĢ ик +ĠамеÑĢик ан +ĠамеÑĢикан Ñģк +ع جب +ãĥĽãĥ¼ãĥł ãĥļãĥ¼ãĤ¸ +Ġник ÑĤо +ĠÙĤ Ùİ +ĠÙĤÙİ Ø§ÙĦ +ĠÙĤÙİاÙĦ Ùİ +ÐIJ ÐĹ +Ùħ جÙħÙĪع +ÙħجÙħÙĪع ات +Ġnecess itÃł +Ġpob li +Ġpobli żu +Ġph ấn +ĠСо обÑī +ÙħÙĤ اط +ÙħÙĤاط ع +Ġ×Ķצ ×ķר×ļ +la ÅŁtırma +ว ิà¸Ķ +วิà¸Ķ ี +วิà¸Ķี à¹Ĥà¸Ń +Ġ그리 ìĬ¤ +Ġ그리ìĬ¤ ëıĦ +ãĤ¿ãĤ¤ ãĥŁ +ãĤ¿ãĤ¤ãĥŁ ãĥ³ãĤ° +ק×ĺ ×Ĵ×ķר +ק×ĺ×Ĵ×ķר ×Ļ×Ķ +Ġ×Ĺ ×ķפ +Ġ×Ĺ×ķפ ש×Ļ +Ø£ جر +Ġим ени +ĠÑĢан ее +à¹Ģà¸ŀืà¹Īà¸Ńà¸Ļ à¹Ĩ +ĠJes ús +Ñģо един +Ñģоедин ен +Ġר ×Ĺ×ķק +à¹Ĥà¸ļ รา +à¹Ĥà¸ļรา à¸ĵ +ĠH Æ¡n +Ġth áºŃp +تع ÙĬÙĬÙĨ +Ġtart Ä±ÅŁ +ĠtartÄ±ÅŁ ma +ĠGes pr +ĠGespr äch +תר ×ķפ +תר×ķפ ×ķת +Ġcat égorie +Ġоказ Ñĭва +ĠналиÑĩ ие +Ġprésent é +Ġk ull +Ġkull and +Ġkulland ı +Ġü nl +Ġünl ü +ĠÙģ Ùĥرة +из аÑĤоÑĢ +×IJ ×ķ׳ +×IJ×ķ׳ ×Ļ×ij +×IJ×ķ׳×Ļ×ij רס +×IJ×ķ׳×Ļ×ijרס ×Ļ×ĺת +ĠÑĢаÑģÑģ маÑĤ +ĠÑĢаÑģÑģмаÑĤ ÑĢ +ĠÑĢаÑģÑģмаÑĤÑĢ ива +تÙĥÙĦ Ùħ +Ùĥت رÙĪ +ÙĥترÙĪ ÙĨÙĬ +ĠÑģо ÑĩеÑĤ +ĠÑģоÑĩеÑĤ а +ãĤĴè¦ĭ ãģĽ +Ġng ừa +ĠÐł еÑģп +ĠÐłÐµÑģп Ñĥб +ĠÐłÐµÑģпÑĥб лик +ãĤ¦ ãĤ© +ãĤ¦ãĤ© ãĥ¼ +ĠÐľ еждÑĥ +ĠìŀĪ ê²Į +Ġm â +ĠìļĶ ì²Ń +ض ار +ลุ à¹īà¸Ļ +ëĮĢ íķĻêµIJ +×ĸ ×Ļ׼ +×ĸ×Ļ׼ ר×ķף +ãĤ¹ ãĥļ +ãĤ¹ãĥļ ãĥ¼ãĤ¹ +ĠкÑĢаÑģ оÑĤ +ï¼ ¨ +ê¼ Ń +ãĤĴ éĽĨ +ãĤĴéĽĨ ãĤģ +ë° Ŀ +Ġ×Ķ׳ ×IJ +Ġ×Ķ׳×IJ ש×Ŀ +Ġê°Ģ ìļ´ +Ġê°Ģìļ´ ëį° +تÙĥÙĦ ÙģØ© +ĠØŃ ÙĤÙĬÙĤÙĬ +Ġh alk +Ġhalk ın +ÑİÑī ÑĥÑİ +ĠÑģп ин +סר×ĺ ף +ĠпеÑĢв ого +Ġпол ож +Ġполож иÑĤелÑĮн +Ġд л +Ġдл иÑĤелÑĮн +ĠV Ä©nh +ê´ ´ +ĠÑģÑĭ ÑĢ +ĠíĨµ íķĺìŬ +ë³ij ìĽIJ +à¹Ĥรà¸ĩ à¸ĩาà¸Ļ +รัà¸ļ à¸ľà¸´à¸Ķ +รัà¸ļà¸ľà¸´à¸Ķ à¸Ĭà¸Ńà¸ļ +تج ÙĨب +s ÅĤ +sÅĤ uch +ãĤ¢ãĥ« ãĥIJ +ãĤ¢ãĥ«ãĥIJ ãĥł +ëī´ ìĬ¤ +Ġpat ië +Ġpatië nt +Ġìĺ ¤í +Ġìĺ¤í ŀ +Ġìĺ¤íŀ Ī +Ġìĺ¤íŀĪ 볤 +ĠDer ne +ĠDerne ÄŁi +wró ci +wróci Äĩ +Ġоб Ñī +ĠобÑī еÑģÑĤв +ĠобÑīеÑģÑĤв енно +ĠêµIJ ìĪĺ +tıģ ımız +Ġ×Ķ×ŀש ×Ļ×ij +k örper +Ġпозв ол +Ġпозвол иÑĤ +ĠChi ến +أخ ÙĪ +ĠAy dın +à¸Ķà¹īาà¸Ļ ล +à¸Ķà¹īาà¸Ļล à¹Īาà¸ĩ +Ġdr u +Ġdru ż +Ġdruż yn +Ġë°ľ íijľ +ĠTh ảo +جÙĩ اد +à¸ģระà¸Ĺ ูà¹ī +Ġк ÑĢов +ĠкÑĢов и +Ġiçer ik +Ġnad zie +Ġnadzie jÄĻ +ĠС моÑĤÑĢ +Ġph ức +ج تÙħاع +جتÙħاع ÙĬØ© +ком пон +компон енÑĤ +Ġб ил +Ġбил еÑĤ +ãĥIJ ãĥ³ãĥī +ĠPol ÃŃcia +اÙĦ تÙĩ +اÙĦتÙĩ اب +ØŃر Ùģ +ت خط +تخط ÙĬØ· +ãĤ³ ãĥ¼ãĥ +ãĤ³ãĥ¼ãĥ Ĵ +ãĤ³ãĥ¼ãĥĴ ãĥ¼ +･･ ï½¥ +à¸ĭ à¸Ńย +Ġcréd it +è²· ãģ£ãģŁ +ĠпоÑĢ Ñıд +ĠпоÑĢÑıд ке +Ġph ó +Ġw ida +Ġwida Äĩ +جر ائÙħ +à¸ľ ี +ĠbÄĻd ÄĻ +Ġ×ŀ פת×Ĺ +ãĥij ãĥ¼ãĥ +ãĥijãĥ¼ãĥ Ĩ +ãĥijãĥ¼ãĥĨ ãĤ£ +ãĥijãĥ¼ãĥĨãĤ£ ãĥ¼ +ĠKa ż +ĠKaż dy +ĠнеобÑħодим оÑģÑĤи +à¸Ł à¸Ńรà¹Į +à¸Łà¸Ńรà¹Į ม +Ġмал ÑĭÑĪ +Ġпл оÑĤ +ĠÑĥ ÑģÑĤÑĢой +ĠÑĥÑģÑĤÑĢой ÑģÑĤва +à¸ĸ à¸Ńà¸Ļ +ĠoluÅŁtur ul +ĠÅĽwi ad +ĠÅĽwiad om +Ùħع Ùĩد +ĠпÑĢоиз веден +Æ ł +ר ×Ļש +Ùħست Ø« +Ùħستث Ùħر +׳×Ļ ×Ļר +pa ñ +Ġ; -) +Ġë°ľ 견 +Ġgör üyor +Ùħؤ ÙĦÙģ +ĠÄIJ á»ģ +ĠاÙĦÙĨ ÙĪاب +×Ĺק ×Ļר×Ķ +Ġm á»ıi +è¿° ãģ¹ +ÐĿ ик +ìŀĸ ìķĦ +ìŀĸìķĦ ìļĶ +prowadzi ÅĤ +l óg +lóg ica +פס ×ĺ +פס×ĺ ×Ļ×ij׾ +Ġ×ŀ ×ĵ×Ķ +Ġ×ŀ×ĵ×Ķ ×Ļ×Ŀ +ãģĵãģĵ ãģ¾ãģ§ +×Ķ ת×Ĺ +×Ķת׊׾×Ķ +Ġפ ×ķס +Ġפ×ķס ×ĺ×Ļ×Ŀ +Ġн ев +Ġнев оз +Ġневоз можно +ĠdostÄĻp ny +Ġغ اÙĦ +ĠغاÙĦ ب +Ġbez pieczeÅĦst +ĠbezpieczeÅĦst wa +åĪĨ ãģĭãĤĭ +ĠF ührung +à¸ģ ีà¹ī +gem Ã¤ÃŁ +à¸Ĭà¹Īวà¸ĩ à¹Ģวลา +Ġìļ°ë¦¬ ëĤĺ +Ġìļ°ë¦¬ëĤĺ ëĿ¼ +ãģ¥ ãģıãĤĬ +ĠاÙĦÙħ سÙĦ +ĠاÙĦÙħسÙĦ ØŃØ© +Ġlibert é +клÑİÑĩ ение +Ġzam ów +Ġzamów ienia +รà¸ĸ à¹Ħà¸Ł +Ø£ ÙģÙĦ +Ø£ÙģÙĦ اÙħ +Ùħ راج +Ùħراج عة +Ġë¹Ħ êµIJ +ĠاÙĦت اب +ĠاÙĦتاب عة +Ġë§Į ëĤĺ +Ġб Ñĥм +ĠбÑĥм аг +Ġgé nero +Ġìŀĺ 못 +×ŀ פ×ķר×ĺ +è²·ãģĦ çī© +ĠÙĦدÙĬ Ùĥ +Ġ×ľ×¢ ×Ļת +Ġ×ľ×¢×Ļת ×Ļ×Ŀ +ĠsÅĤ ab +ĠпÑĢедÑģÑĤав лÑı +ãĤ¿ ãĤ¤ãĥĪ +ãĤ¿ãĤ¤ãĥĪ ãĥ« +Ùħ ص +Ùħص Ø·Ùģ +ÙħصطÙģ Ùī +Ġdifficult é +ãĥĨãĤ£ ãĥĸ +Ġpew noÅĽci +ĠpewnoÅĽci Äħ +Ġ무 ìĬ¨ +Ø¥ رس +إرس اÙĦ +Ġд алÑĮ +ĠдалÑĮ ÑĪе +Ġ׾ ×ł×¡ +Ġ×ľ×ł×¡ ×ķת +หมูà¹Ī à¸ļà¹īาà¸Ļ +×ŀס×ŀ ׼×Ļ +أسÙĦ ÙĪب +Ġzw ÅĤ +ĠzwÅĤ as +ĠzwÅĤas zc +ĠzwÅĤaszc za +ĠпÑĢ еж +ĠпÑĢеж де +ĠоÑĢганиз аÑĨиÑı +Ġdön emin +Ġdönemin de +Ġ Ủ +ĠỦ y +ä¸ĭ ãģĴ +ĠпоÑģлед ние +Ġgü ne +Ġgüne ÅŁ +Ġ×IJ ×ĸר +Ġ×IJ×ĸר ×Ĺ×Ļ +ãģ§ãģĤ ãĤįãģĨ +ĠÙĨ ÙĤ +ĠÙĨÙĤ اط +æŃ£ ãģĹãģĦ +ĠÑĢ ег +ĠÑĢег иона +ĠFör der +ê²½ ìĺģ +dıkl ar +dıklar ını +trzym aÄĩ +أش Ùĥ +أشÙĥ اÙĦ +×Ķת ×IJ +×Ķת×IJ ×ŀ×Ķ +à¸Ĺำà¹ĥหà¹ī à¹Ģà¸ģิà¸Ķ +ĠGeb ä +ĠGebä ude +ĠСеÑĢ г +ĠСеÑĢг ей +Ġз доÑĢов +ĠздоÑĢов ÑĮÑı +Ġr ãi +ĠпÑĢед ÑĥÑģ +ĠпÑĢедÑĥÑģ моÑĤÑĢ +ĠпÑĢедÑĥÑģмоÑĤÑĢ ен +Ġ×Ķצ ×Ļ×ij +Ġ×Ķצ×Ļ×ij ×ķר×Ļ +Ġdés ir +Ġн оÑĩ +ĠноÑĩ ÑĮ +möglich keiten +Ġ×IJ×Ĺר ×ķ׳×Ļ×Ŀ +Ġsoir ée +ĠNh áºŃn +Ù ª +à¸Ľà¸£à¸°à¸§à¸±à¸ķิ ศาสà¸ķรà¹Į +êµIJ íĨµ +ĠØ£ Ø®ÙĬ +Ġdé cid +Ġdécid é +Ġwy ja +Ġwyja ÅĽni +Ġ สิ +Ġสิ à¸ĩ +Ġสิà¸ĩ หา +Ġสิà¸ĩหา à¸Ħม +à¹ģ à¸Ńรà¹Į +หà¸Ļà¹īา à¸Īà¸Ń +ס תר +Ġê ¶ +Ġê¶ Į +Ġê¶Į 리 +pl ätze +ب Ø·ÙĦ +ê±´ ìĦ¤ +Ġ×IJ ×Ļ×ŀ×Ļ +Ġ×IJ×Ļ×ŀ×Ļ ×Ļ׾ +ãģ ½ +تر اث +×IJ׾ ×Ļ×ŀ×ķת +Ġdispon ÃŃveis +Ġz ale +Ġzale ży +à¸Ľà¸£à¸°à¸Ĭา สัมà¸ŀัà¸Ļà¸ĺà¹Į +ĠÅļw iat +Ġpor ówn +Ġporówn a +Ġ׾×ĺ ×ķ×ijת +×Ķ×ĸ ×ŀ׳×Ķ +Ġ×Ľ×ª ×ķצ×IJ×Ķ +Ġ×ij ק׾ +Ġ×ijק׾ ×ķת +ĠоÑĤ кÑĢ +ĠоÑĤкÑĢ Ñĭва +ãĥij ãĥ¯ãĥ¼ +ë¿IJ ë§Į +Ġв ÑģÑı +ĠвÑģÑı к +ãģ¨ãģª ãģ£ãģ¦ãģĦãĤĭ +Ġgi áºŃn +Ġок ÑĢÑĥ +ĠокÑĢÑĥ жа +ĠокÑĢÑĥжа ÑİÑī +ĠUnivers ität +ĠÑĢ ож +ĠÑĢож д +ĠÑĢожд ениÑı +Ø® ÙĬÙĦ +Ġкомпани й +ĠÑĢазлиÑĩ нÑĭе +ĠЦ ена +׳×Ļ ×ķ×ĸ +׳×Ļ×ķ×ĸ ׾ +׳×Ļ×ķ×ĸ׾ ×ĺר +Ġê³µ ê°Ħ +Ġê°ľ ëħIJ +landır ma +ĠÑĥдал ен +à¸ŀัà¸ģ à¸ľ +à¸ŀัà¸ģà¸ľ à¹Īà¸Ńà¸Ļ +Ġprote cción +Ġb ÅĤ +ĠbÅĤ ÄĻd +à Ī +Ġíĸī ë³µ +ĠÅŁ ü +ĠÅŁÃ¼ phe +Ġí Ķ +ĠíĶ ¼ +ĠíĶ¼ íķ´ +Ġëĭ¤ 르 +à¹Ħมà¹Ī à¹Ģà¸ģิà¸Ļ +ãģ¿ ãģª +ãģ¿ãģª ãģķãĤĵ +ĠпоÑĤ ÑĢеб +ĠпоÑĤÑĢеб иÑĤел +ĠاÙĦÙĥÙĦ اÙħ +ìķĦ ë²Ħ +ìķĦë²Ħ ì§Ģ +ãĤĴ使 ãģ£ãģŁ +Ġbụ i +ĠпоÑĤ еÑĢ +ĠпоÑĤеÑĢ Ñı +ĠØ¢ ÙĦاÙģ +ĠнаÑģÑĤоÑıÑī ее +ãģıãģªãĤĬ ãģ¾ãģĹãģŁ +clus ão +ãĤ³ ãĥĶãĥ¼ +צ פ×Ļ +צפ×Ļ ×Ļ×Ķ +Ø® ÙĦا +Ø®ÙĦا ص +ล à¹īำ +ãĥ¯ ãĤ¤ãĥ³ +Ġมี à¸Ļา +Ġมีà¸Ļา à¸Ħม +Ø´ خص +شخص ÙĬات +Ġ×ĸ ק +Ġ×ĸק ×ķק +×Ļ ×Ļצ +×Ļ×Ļצ ×Ĵ +èĢĥãģĪ æĸ¹ +Ġürün ü +ĠиÑģп ол +ĠиÑģпол ни +Ġcompañ ero +ק צ×Ķ +×ŀ×¢ ׳×Ļק +Ùħ ØŃÙħد +Ġc ámara +Ġп ед +Ġпед аг +Ġпедаг ог +м аÑĢ +маÑĢ к +×Ķת ׳×Ĵ×ĵ +ĠìĨĮ ê°ľ +Ġcom unitÃł +ê³ ¤ +ĠNg Ãłi +สà¸ĩ à¸ļ +ĠmieszkaÅĦ ców +ĠÙĨ ÙĩائÙĬ +iv ité +Ġи де +Ġиде алÑĮн +ĠØ£ سبÙĪع +Ġ×Ļ ×¢×ľ +Ġ׾ ר×IJש +Ġ׾ר×IJש ×ķ׳×Ķ +ĠзапиÑģ и +ĠкоÑĢ пÑĥÑģ +วà¸ĩ ศ +วà¸ĩศ à¹Į +ĠÐĶ м +ĠÐĶм иÑĤ +ĠÐĶмиÑĤ ÑĢ +Ġkön nt +Ġböl ges +Ġbölges inde +׼ ×Ļ׼ +׼×Ļ׼ ר +ĠاÙĦØ¥ Ø«ÙĨ +ĠاÙĦإثÙĨ ÙĬÙĨ +Ġng á»Ļ +ì¹ ł +د راج +Ġu da +Ġuda ÅĤo +ìº IJ +بر ÙĨاÙħج +ĠÑģÑĥд еб +ĠÑģÑĥдеб н +Ġzun ächst +ĠEduc ación +ãģ¨ãģª ãģ£ãģ¦ãģĦãģ¾ãģĻ +Ġ×Ķ×IJ ×ŀ×Ļת×Ļ +ĠÄ° nt +ĠÄ°nt ernet +ĠcaÅĤ ego +ãĥĹãĥª ãĥ³ +Ø¥ بد +إبد اع +ĠпоÑĢ ÑĤал +à¹Ĥà¸ķ à¹ī +Ġ×Ķק ש×ķר +пл од +ĠÙħ د +ĠÙħد رÙĬد +×ŀסע ×ĵ×Ķ +ĠØ´ÙĬ ئ +ĠØ´ÙĬئ ا +à¸ģà¹Īà¸Ń สรà¹īาà¸ĩ +Ġì°¸ ê³ł +à¹Ģà¸Ĺ ร +à¹Ģà¸Ĺร à¸Ķ +Ġ×ij×ŀ קר×Ļ×Ŀ +Ġb ât +Ġbât iment +åij¼ ãģ³ +ç´ł æķµ +ç´łæķµ ãģª +przedsiÄĻbior st +przedsiÄĻbiorst w +Ġ×ł×ª ×ķ׳×Ļ×Ŀ +×Ĺ׾ ×ķ×Ŀ +ร วย +Ùħ ÙĪضÙĪع +ĠÑģоб ÑĢан +вед ÑĥÑī +ĠÑĤе аÑĤ +ĠÑĤеаÑĤ ÑĢ +m eye +meye ceÄŁi +Ġpien iÄħ +ĠpieniÄħ d +ĠpieniÄħd ze +ÑĢез иденÑĤ +ØŃ ØµØ± +ìĺ ¥ +à¹Ģย ืà¸Ńà¸Ļ +ĠÑĥ ни +ĠÑĥни веÑĢ +ĠÑĥнивеÑĢ Ñģ +ĠÑĥнивеÑĢÑģ иÑĤеÑĤ +ĠاÙĦر ØŃ +ĠاÙĦرØŃ ÙħÙĨ +ĠÑĤеÑħ нолог +ĠÑĤеÑħнолог ии +ìĹIJ ëĦĪ +ìĹIJëĦĪ ì§Ģ +Ġíķ Ń +ĠíķŃ ìĥģ +à¸ĺ า +à¸ĺา à¸ķุ +ĠEspañ ol +×ĵ×Ĵ ש +Ġêµ ī +Ġêµī ìŀ¥ +Ġêµīìŀ¥ íŀĪ +ĠÅĤ at +ĠÅĤat wo +Ġk á»ĭch +Ø¥ ز +إز اÙĦØ© +ĠдейÑģÑĤв ие +ĠsaÄŁ layan +สุà¸Ķ ยà¸Ńà¸Ķ +Ġzosta Äĩ +Ġdispon ÃŃvel +ïº į +ver ständ +verständ lich +tw or +twor zyÄĩ +ع جز +à¹Ģà¸Ĥ à¹īม +ยà¹Ī à¸Ńม +Ġstrat ég +Ġstratég ie +à¸ľà¸¥ à¹Ħมà¹ī +Ġê°ģ ì¢ħ +ĠÙħ ÙĪا +ĠÙħÙĪا ض +ĠÙħÙĪاض ÙĬع +اØŃ ØªØ¬ +اØŃتج اج +Ġ Ấ +ĠẤ n +×ŀ ×ŀש׾×Ķ +ĠÅŁek il +×ŀ ×Ĺ׾ +×ŀ×Ĺ׾ ×ķת +Ġ à¸ĺ +Ġà¸ĺ ัà¸Ļ +Ġà¸ĺัà¸Ļ วา +Ġà¸ĺัà¸Ļวา à¸Ħม +Ġìĭ¤ ìłľ +Ġìĭ¤ìłľ ë¡ľ +ì¤ij ìķĻ +ëįĶ ëĿ¼ +ĠÑĪ иÑĢ +ĠÑĪиÑĢ око +Ġsol ución +วาà¸ĩ à¹ģà¸ľà¸Ļ +×IJ×ķ×ĺ ×ķ×ŀ +×IJ×ķ×ĺ×ķ×ŀ ×ĺ×Ļ +ĠÑĢ еÑģÑĤ +ĠÑĢеÑģÑĤ оÑĢ +ĠÑĢеÑģÑĤоÑĢ ан +ëį ¸ +ÑĤ ÑĢад +ÑĤÑĢад и +ÑĤÑĢади ÑĨион +ÑĤÑĢадиÑĨион н +มะ à¹Ģรà¹ĩ +มะà¹Ģรà¹ĩ à¸ĩ +à¹Ĥ ส +Ġol masını +×ŀ×ķס ר +ĠоÑĤноÑĪ ении +Ġê°ĢëĬ¥ ìĦ± +Ġy uk +Ġyuk arı +ìĨ Ķ +ĠÑģ ÑĦ +ĠÑģÑĦ еÑĢе +Ġק ×ķפ +ãĤ± ãĥ¼ãĤ +ãĤ±ãĥ¼ãĤ Ń +âĢķ âĢķ +ĠاÙĦØ£ ÙĦÙħ +ĠاÙĦØ£ÙĦÙħ اÙĨÙĬ +Ả N +ת×ķ׼ ׳×Ļ×ķת +ĠÑģÑĥÑīеÑģÑĤв ÑĥеÑĤ +æĪij ãĢħ +ĠاÙĦص ادر +ĠTr á»įng +Ġа д +Ġад миниÑģÑĤ +ĠадминиÑģÑĤ ÑĢа +ĠадминиÑģÑĤÑĢа ÑĨи +ĠдÑĢÑĥг ими +Ñģп еÑĪ +عÙĦاÙħ ات +Ġа б +Ġаб Ñģол +ĠабÑģол ÑİÑĤ +ĠабÑģолÑİÑĤ но +ฤ à¸Ķู +é tr +étr anger +нÑı ÑĤи +нÑıÑĤи е +×¢ ×ķ׳ +×¢×ķ׳ ש +ĠÙĤ ائ +ĠÙĤائ ÙĦا +Ġм аÑģ +ĠмаÑģ ло +ãĥī ãĤ¤ +ãĥīãĤ¤ ãĥĦ +å¿ħè¦ģ ãģĮãģĤãĤĬãģ¾ãģĻ +×ŀ×ķ×ĸ ×Ļ×IJ +×ŀ×ķ×ĸ×Ļ×IJ ×ķף +ĠNgo ại +Ġkê nh +à¸ģาร à¸Ńà¸Ńà¸ģà¹ģà¸ļà¸ļ +×ŀ פק +×ŀפק ×ĵ +ÙħÙĨ از +ÙħÙĨاز ÙĦ +ë· ° +íĹ ¤ +ÙħÙĩ ارات +Ġpropri été +פ×Ĵ ×Ļש×Ķ +Ñĩ ÑĢ +ÑĩÑĢ еж +ÑĩÑĢеж ден +×Ķ ×ķצ×IJ×Ķ +ØŃÙĥ ÙĬÙħ +ĠíĻ Ī +ĠíĻĪ íİĺìĿ´ì§Ģ +åİ ³ +åݳ ãģĹãģĦ +×¢ ×ŀ×ĵ×Ķ +ĠAu ÃŁen +سÙĪ Ø¡ +ë¹ Ī +ĠÙĪ Ø® +ĠÙĪØ® اصة +ин ÑĤеÑĢ +инÑĤеÑĢ еÑģ +èĩ´ ãģĹãģ¾ãģĻ +Ġhük üm +à¹Ħà¸Ĥ มัà¸Ļ +Ġdav ran +Ġdavran Ä±ÅŁ +à¹Ģà¸ķ ียà¸ĩ +в ÑĢем +вÑĢем енно +à¹Ģà¸Ĺศ à¸ģา +à¹Ģà¸Ĺศà¸ģา ล +å¼ķ ãģ£ +å¼ķãģ£ è¶ĬãģĹ +×IJר ×ķ×Ĺ +×IJר×ķ×Ĺ ×ª +à¹Ģ วิ +à¹Ģวิ รà¹Į +à¸Ńยà¹Īาà¸ĩ รวà¸Ķà¹Ģรà¹ĩว +ĠìŬ íĸī +ĠÑĢан ÑĮ +ĠÑĢанÑĮ ÑĪе +Ġzob ow +Ġzobow iÄħ +ĠzobowiÄħ z +Ġ×ķ׼ ×ŀ×ķ×ijף +ĠاÙĦÙħ Ùĩ +ĠاÙĦÙħÙĩ ÙĨÙĬ +ãĤ¢ ãĤ¸ +ãĤ¢ãĤ¸ ãĤ¢ +ë°© ìĨ¡ +à¸Ńà¸Ńà¸ģ à¸ģำลัà¸ĩ +à¸Ńà¸Ńà¸ģà¸ģำลัà¸ĩ à¸ģาย +am éli +améli orer +å½ĵãģŁãĤĬ åīį +Ġreg elm +Ġregelm Ã¤ÃŁig +ãģĬ åĭ +ãģĬåĭ § +ãģĬåĭ§ ãĤģ +Ġm Æ°á»Ŀi +بر Ùħج +ĠNat ürlich +ĠD Å©ng +ĠاÙĦر جاÙĦ +Ġthé p +Ġol muÅŁtur +×ŀ×ķס ×Ļק×Ķ +f älle +주 íĥĿ +ĠاÙĦÙģ رص +Ġnaj wiÄĻks +ĠnajwiÄĻks zy +Ġça ÄŁ +ĠçaÄŁ rı +ì¸ ł +ĠvÃŃ ct +ĠvÃŃct ima +ĠÑģовеÑĢ ÑĪен +×Ķ×Ļ ×Ļת×Ļ +à¹Ģà¸Ķ ี +à¹Ģà¸Ķี à¹ĭ +à¹Ģà¸Ķีà¹ĭ ยว +ü yü +Ġд оп +Ġдоп олн +Ġдополн иÑĤелÑĮно +à¹ģà¸ķà¸ģà¸ķà¹Īาà¸ĩ à¸ģัà¸Ļ +Ġá l +Ġál bum +à¸Ľà¸£à¸°à¸Īำ à¸Ľà¸µ +ĠÑĦ едеÑĢ +ĠÑĦедеÑĢ алÑĮн +Ġobs ÅĤ +ĠobsÅĤ ugi +à¹Ģร ืà¹Ī +à¹Ģรืà¹Ī à¸Ńย +à¹Ģรืà¹Īà¸Ńย à¹Ĩ +ëģ Į +Ġngh ìn +ĠBaÅŁkan lıģı +تأ سÙĬ +تأسÙĬ س +Ġ×ij×ij ×ķקר +Ġ×¢×ij×ķ×ĵ ×ķת +Ġبص ÙĪرة +ãĤıãģij ãģ§ãģ¯ãģªãģĦ +führ er +ãĤ¹ ãĤŃ +ãĤ¹ãĤŃ ãĥ« +ĠاÙĦÙĤ ض +ĠاÙĦÙĤض ÙĬØ© +Ġдолж ноÑģÑĤ +Ùģار ÙĤ +Ġcomeç ou +Ġorganis é +Ġxu ân +ĠÑģообÑī аеÑĤ +ĠпÑĢи д +ĠпÑĢид еÑĤÑģÑı +TÃľ RK +ãĥ¬ ãĥ¼ãĤ·ãĥ§ãĥ³ +Kh ông +است Ùģ +استÙģ ادة +ä¸ĬãģĮ ãģ£ãģ¦ +Ġum ie +Ġumie jÄĻ +ĠumiejÄĻ tn +ĠumiejÄĻtn oÅĽci +ëĤ ¸ +à¹Ģà¸Ļ à¸Ńรà¹Į +×ĵ×ķ ×ķ×Ĺ +ÃŃs imo +I ÃĬ +IÃĬ N +Ġalcan ç +Ġ à¸ķุ +Ġà¸ķุ ลา +Ġà¸ķุลา à¸Ħม +ש׾ ×ĺ×ķף +Ġél è +Ġélè ves +ĠÄij u +ĠÄiju á»ķi +ĠØ£ Ùģ +ĠØ£Ùģ رÙĬ +ĠØ£ÙģرÙĬ ÙĤÙĬ +ĠØ£ÙģرÙĬÙĤÙĬ ا +ãĤĴæİ¢ ãģĻ +ĠпÑĢед ложениÑı +ج اد +ĠÑħоÑĤ ÑĮ +Ñģ ал +Ñģал он +à¸Ľà¸£à¸° à¹Ģม +à¸Ľà¸£à¸°à¹Ģม ิà¸Ļ +ãĤŃ ãĥĥãĥģ +ãĤŃãĥĥãĥģ ãĥ³ +×ij×ĵ×Ļק ×ķת +Ġch ù +Ġchù a +ÐĴ иде +ÐĴиде о +иÑĢов ка +ĠÑħоÑĤ иÑĤе +Ġspéc ifique +รส à¸Ĭาà¸ķิ +è¾¼ ãĤĵãģł +伸 ãģ³ +×Ķצ׾ ×Ĺת +ãģ©ãģ® ãĤĪãģĨãģ« +سع ادة +Ġл ид +Ġлид еÑĢ +ม à¸ĩ +มà¸ĩ à¸Ħล +ØŃ Ø§ÙħÙĦ +หล ุà¸Ķ +à¸Ńยà¹Īาà¸ĩ à¸ķà¹Īà¸Ń +à¸Ńยà¹Īาà¸ĩà¸ķà¹Īà¸Ń à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ +ãģķãģĽãģ¦ éłĤ +تس ÙĪÙĬ +تسÙĪÙĬ ÙĤ +ĠaÅŁaģı d +ĠaÅŁaģıd aki +ĠÑĨ елÑĮ +ĠÑĨелÑĮ Ñİ +ĠAra ÅŁtırma +à¸Ĥัà¸ļ รà¸ĸ +Ùĩ Ø°Ùĩ +ลà¸ĩ à¸Ĺะ +ลà¸ĩà¸Ĺะ à¹Ģà¸ļ +ลà¸ĩà¸Ĺะà¹Ģà¸ļ ียà¸Ļ +تÙĥ اÙħÙĦ +Ġc io +Ġcio è +ãģ¦ ãģĬãģı +ĠاÙĦصØŃ ÙģÙĬ +ĠíĬ¹ ìłķ +полн иÑĤÑĮ +ãĤĵ ãģĺãĤĥãģªãģĦ +ãĤĵãģĺãĤĥãģªãģĦ ãģĭ +ĠاÙĦج Ùĩ +ĠاÙĦجÙĩ ات +ĠÑĥÑģпеÑĪ но +Ġв ок +Ġвок ÑĢÑĥг +ĠÑģиÑĤÑĥ аÑĨиÑı +Ġ×Ķ×IJ ×ŀר +Ġ×Ķ×IJ×ŀר ×Ļק +Ġ×Ķ×IJ×ŀר×Ļק ×IJ×Ļ +×ŀ ×Ĵ×ĸ +×ŀ×Ĵ×ĸ ×Ļף +Ġак ÑĤÑĥ +ĠакÑĤÑĥ алÑĮн +é ta +éta is +Ġmog ÅĤa +ĠÑĤоÑĩ ки +Ġ×ŀ×Ķ ×ŀ×¢ +Ġ×ŀ×Ķ×ŀ×¢ ×¨×Ľ×ª +มี à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺà¸´à¸łà¸²à¸ŀ +×Ļר ×Ļ×ĵ×Ķ +×Ĵר ×ŀ׳ +×Ĵר×ŀ׳ ×Ļ×Ķ +Ġг лав +Ġглав ное +Ġ미 ëŀĺ +Ġ׳׼ ×ķ׳×Ķ +ĠÙĪ Ø·ÙĨÙĬ +op port +opport unitÃł +Ġh ủy +ĠÙĦ تØŃ +ĠÙĦتØŃ ÙĤÙĬÙĤ +Ġó rg +Ġórg ão +ãĤ¹ ãĥĶ +ãĤ¹ãĥĶ ãĥ¼ãĥī +Ġön ü +Ġönü ne +Ùħع اÙħÙĦ +ש×ŀ ×Ļר×Ķ +ĠвеÑģÑĮ ма +ĠwiÄĻks zo +ĠwiÄĻkszo ÅĽÄĩ +Ġاست راتÙĬج +ĠاستراتÙĬج ÙĬØ© +ĠÙģ Ø¥ +ĠÙģØ¥ ذا +à¹Ģà¸Ĭืà¹Īà¸Ń ม +à¹Ģà¸Ĭืà¹Īà¸Ńม à¸ķà¹Īà¸Ń +Ġ׾ פר +Ġ׾פר ×ĺ×Ļ×Ŀ +Ùħض ÙĬ +ĠGer çek +Ġçocuk ların +ÙĪØ« ائÙĤ +ĠÙħساء Ùĭ +Ġunterstüt zt +Ġpré st +Ġprést amo +ĠÐłÐ°Ð· меÑĢ +ĠÅŁ eker +Ġsé culo +×ij×Ķ ×Ļר +Ø´Ùĩ ÙĪر +Ġ à¸Ńีà¸ģ +Ġà¸Ńีà¸ģ à¸Ĺัà¹īà¸ĩ +Ġlleg ó +à¸¨à¸´à¸¥à¸Ľ ะ +æĪij ãģĮ +æĪijãģĮ 家 +ع ÙĤÙĪ +عÙĤÙĪ بات +ĠF älle +Ġs ÅĤuż +ĠsÅĤuż b +ĠاÙĦØŃÙĤ ÙĪÙĤ +Ġпл иÑĤ +Ġи ноÑģÑĤ +ĠиноÑģÑĤ ÑĢан +ĠиноÑģÑĤÑĢан н +à¹ĥà¸Ļ à¸Ĥà¸ĵะà¸Ĺีà¹Ī +ãĤ« ãĥĨ +ãĤ«ãĥĨ ãĤ´ +ãĤ«ãĥĨãĤ´ ãĥª +à¸Ńิ ส +à¸Ńิส ระ +à¹Ģà¸ľà¸¢ à¹ģ +à¹Ģà¸ľà¸¢à¹ģ à¸ŀร +à¹Ģà¸ľà¸¢à¹ģà¸ŀร à¹Ī +ãģĬ ãģĦ +ãģĬãģĦ ãģĹãģĦ +است ÙĤÙĦ +استÙĤÙĦ اÙĦ +تØŃ Ø¶ +تØŃض ÙĬر +åĬ© ãģij +Ùħر اÙģÙĤ +Ġ×ĵ ×ķר +Ġ×ĵ×ķר ש +×ŀת×Ļ ×Ļ×Ĺס +ס ×Ļ׼ +ס×Ļ׼ ×ķ×Ŀ +íĮĮ íĬ¸ +Ġwy ÅĽ +ĠwyÅĽ w +ĠwyÅĽw iet +ĠwyÅĽwiet l +ĠاÙĦاÙĨ ساÙĨ +ĠStra ÃŁen +ï¼ ¬ +ãģ« åŁº +ãģ«åŁº ãģ¥ +Ġcap ÃŃtulo +ลุ ย +Ġ×Ķ×ŀק צ×ķ×¢×Ļ +ãģĤãĤĭ ç¨ĭ度 +á» ¢ +ĠاÙĦ ÙĦا +ĠاÙĦÙĦا زÙħØ© +æķĻ ãģĪ +Ġרש ×IJ×Ļ +з ав +зав иÑģ +завиÑģ им +à¸Ľà¸±à¸Ī à¸Īัย +à¹Ģà¸ĭ ล +à¹Ģà¸ĭล ลà¹Į +Ġdiffé rence +ĠAlt ın +Ġк ÑĢай +ĠкÑĢай не +Ġз ло +Ġgün ümüz +Ġн аÑĤÑĥÑĢ +ĠнаÑĤÑĥÑĢ алÑĮн +×Ĵ×ķ׾ ש×Ļ×Ŀ +Ġк аÑĤегоÑĢ +ĠкаÑĤегоÑĢ ии +Ġз нак +à¸ģà¹Īà¸Ńà¸Ļ หà¸Ļà¹īา +à¸ģà¹Īà¸Ńà¸Ļหà¸Ļà¹īา à¸Ļีà¹ī +ĠÙħÙĨ ت +ĠÙħÙĨت خب +ãĥĽ ãĥ¼ãĥ« +Ġе вÑĢо +ส ว +สว ม +ĠìľĦ ìĽIJ +ĠìľĦìĽIJ ëĭĺ +ĠاÙĦØŃ ÙĪØ« +ĠاÙĦØŃÙĪØ« ÙĬ +ĠÑģодеÑĢж иÑĤ +ãĥķãĤ¡ ãĥĥãĤ·ãĥ§ãĥ³ +Ġ à¸ģัà¸Ļ +Ġà¸ģัà¸Ļ ย +Ġà¸ģัà¸Ļย ายà¸Ļ +ãĤª ãĥª +ãĤªãĥª ãĤ¸ +ãĤªãĥªãĤ¸ ãĥĬãĥ« +Ġб ÑĢенд +ãĤĴæĮģ ãģ£ãģ¦ãģĦãĤĭ +Ġinvers ión +Ġê° ĸ +Ġê°ĸ ê³ł +Ġnov itÃł +ê´Ģ ê´ij +Ġà¸ŀ ฤษ +Ġà¸ŀฤษ à¸łà¸² +Ġà¸ŀà¸¤à¸©à¸łà¸² à¸Ħม +×ķר ×Ĺ×Ļ×Ŀ +׼׾ ×ķ׾ +Ġng ạc +×Ļ ×Ļש +×Ļ×Ļש ×ķ×ij +f äll +fäll ig +ĠÑĤÑĢеб ÑĥеÑĤÑģÑı +Ġcar á +Ġcará cter +Ġprinc ÃŃpio +ĠÅĤ az +ĠÅĤaz ien +ĠÅĤazien k +Ġgi ãn +ÑģÑĤÑĢа ива +Ùħس اب +Ùħساب ÙĤØ© +à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ à¸Ķืà¹Īม +ترÙĥ ÙĬب +vol ução +ĠÐŁ оÑĩ +ĠÐŁÐ¾Ñĩ ем +ĠÐŁÐ¾Ñĩем Ñĥ +казал оÑģÑĮ +ĠпÑĢимен ениÑı +à¹Ģà¸Ĺ ียม +íĮ Ķ +à¸Ĥà¹īà¸Ń à¹Ģสà¸Ļà¸Ń +à¸Ľà¸±à¸į à¸įา +Ġоб ÑĥÑĩ +ĠобÑĥÑĩ ениÑı +ĠÑģеÑĢ и +ĠÑģеÑĢи ал +Ġingl és +ĠÙĦ Ùĥرة +Ġ×ĺ ׾ +Ġ×ĺ׾ פ×ķף +Ġìł ij +Ġìłij ê·¼ +×IJ ×ķ×Ĵ +×IJ×ķ×Ĵ ×ķס +×IJ×ķ×Ĵ×ķס ×ĺ +ĠболÑĮÑĪ ое +ĠÐļон еÑĩно +×¢×Ļת ×ķ׳ +×¢×Ļת×ķ׳ ×IJ×Ļ +Ġкноп к +Ġз н +Ġзн аÑĤÑĮ +ĠÄij á»± +ĠÄijá»± ng +вл аж +влаж н +×ŀ ×Ļ×ĺ×ij +ãĤ¬ ãĤ¤ +ãĤ¬ãĤ¤ ãĥī +........ .. +Ġà¸ģ ุม +Ġà¸ģุม à¸łà¸²à¸ŀ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀ ัà¸Ļ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļ à¸ĺ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļà¸ĺ à¹Į +be z +bez pieczeÅĦst +bezpieczeÅĦst w +ãĥijãĥij æ´» +ع اط +عاط Ùģ +ĠÄij áºŃm +Ġз ÑĢ +ĠзÑĢ ениÑı +Ġbor ç +Ġнед ел +Ġнедел Ñİ +Ġh á»ı +Ġhá»ı ng +ìŀ¥ ìķł +ìŀ¥ìķł ìĿ¸ +ĠاÙĦع ÙĦاÙĤØ© +Ġíģ ¬ +Ġíģ¬ ê²Į +à¹Ħร à¹Ī +à¸ļา à¸Ķ +à¸ļาà¸Ķ à¹Ģà¸Īà¹ĩà¸ļ +à¸Ŀ รั +à¸Ŀรั à¹Īà¸ĩ +à¸Ŀรัà¹Īà¸ĩ à¹Ģศ +à¸Ŀรัà¹Īà¸ĩà¹Ģศ ส +ר ×¢×Ļ +רע×Ļ ×ķ׳×ķת +Ġë Į +ĠëĮ ĵ +ĠëĮĵ ê¸Ģ +Ġnaj b +Ġnajb li +Ġnajbli ż +Ġnajbliż sz +ĠиÑģполÑĮз ÑĥеÑĤÑģÑı +Ġcient ÃŃf +ĠcientÃŃf ico +×¢ ×ŀק +Ġg ợi +Ø´ ØŃÙĨ +ĠÅĽ m +ĠÅĽm ier +ĠÅĽmier ci +à¸Ħาสิà¹Ĥà¸Ļ à¸Ńà¸Ńà¸Ļà¹Ħลà¸Ļà¹Į +×Ĺש×ij ת×Ļ +Ġn ingu +Ġningu ém +è¾¼ ãĤģ +ãģ · +ĠÑĥ г +ĠÑĥг ол +ï½ ° +פת ×Ļ×Ĺ +פת×Ļ×Ĺ ×ª +Ġ×Ķר×IJש ×ķ׳×Ļ×Ŀ +p ósito +ãĤŃ ãĥ¬ãĤ¤ +ãģ© ãģĵãĤį +à¹Ģà¸Ĺà¹Īา à¹Ħ +à¹Ģà¸Ĺà¹Īาà¹Ħ หร +à¹Ģà¸Ĺà¹Īาà¹Ħหร à¹Ī +ĠинÑĤеÑĢ ÑĮеÑĢ +ĠØŃ Ø§Ø¬ +ĠØŃاج Ø© +สี à¸Ĥาว +ìĸ ¼ +Ġn á»Ļ +Ġná»Ļ p +ĠÃŃ nd +ĠÃŃnd ice +สำ รวà¸Ī +Ġкажд ой +Ġhot éis +Ġnast ÄĻ +ĠnastÄĻ pn +Ġ×Ķק ×ķ×ĵ +Ġ×Ķק×ķ×ĵ ×Ŀ +פ ×ķפ +פ×ķפ ×ķ׾ +פ×ķפ×ķ׾ ר×Ļ +вÑĪ ей +ãĤ·ãĥ³ ãĥĹ +ãĤ·ãĥ³ãĥĹ ãĥ« +ĠzdjÄĻ Äĩ +ĠгÑĢÑĥпп а +Ġпом еÑī +ĠпомеÑī ениÑı +ãģ©ãģĨ ãģĦãģĨ +ĠиÑģп ÑĭÑĤа +Ġog ÅĤ +ĠogÅĤ os +ĠogÅĤos zen +ĠogÅĤoszen i +สรà¹īาà¸ĩ สรร +สรà¹īาà¸ĩสรร à¸Ħà¹Į +à¸ŀร รà¸ĵ +Ġçık Ä±ÅŁ +ĠÑĩаÑģÑĤ ноÑģÑĤи +Ġ×ķ ×Ļ×ķתר +ç¶ļãģį ãĤĴ +ç¶ļãģįãĤĴ èªŃ +ç¶ļãģįãĤĴèªŃ ãĤĢ +à¸ģร ั +à¸ģรั ม +г ÑĢаÑĦ +Ġв лад +Ġвлад елÑĮ +ĠвладелÑĮ ÑĨ +Ġistedi ÄŁ +ĠistediÄŁ iniz +×ij׾ ×¢ +×ij×ľ×¢ ×ĵ×Ļ +ÙħÙĪ اÙģ +ÙħÙĪاÙģ ÙĤØ© +Ġ×Ļ ×ķר +Ġ×Ļ×ķר ק +ãĤ«ãĥ¼ãĥī ãĥŃãĥ¼ãĥ³ +ĠاÙĦÙħØ´ ÙĥÙĦ +ĠاÙĦÙħØ´ÙĥÙĦ Ø© +ĠêµŃ íļĮ +ס פ×ĺ +ספ×ĺ ×ŀ +ספ×ĺ×ŀ ×ijר +Ġìĸ´ ëłµ +Ùĥ اÙħ +ÙĥاÙħ ÙĬرا +sch lü +schlü sse +ĠØ« ÙĨ +ĠØ«ÙĨ ائÙĬ +ìī ½ +ĠÐŀ Ñģоб +ĠÐŀÑģоб енно +Ġин веÑģÑĤи +ĠинвеÑģÑĤи ÑĨи +اØŃ ØªÙħ +اØŃتÙħ اÙĦ +E Äŀ +EÄŀ Ä° +íķĺ ê²łëĭ¤ +Ġ×IJ ×ijר×Ķ +Ġ×IJ×ijר×Ķ ×Ŀ +Ġ×ij×Ĺ ×Ļ׳×Ŀ +Ø£ ÙĪض +Ø£ÙĪض اع +Ġdé l +Ġdél ai +Ġ×IJ×ķ×Ķ ×ij×Ļ×Ŀ +ĠÑģо Ñħ +ĠÑģоÑħ ÑĢ +ĠÑģоÑħÑĢ ани +ĠдоÑģÑĤ иж +ĠдоÑģÑĤиж ени +สิà¹Īà¸ĩ à¹ģ +สิà¹Īà¸ĩà¹ģ วà¸Ķ +สิà¹Īà¸ĩà¹ģวà¸Ķ ล +สิà¹Īà¸ĩà¹ģวà¸Ķล à¹īà¸Ńม +ĠاÙĦÙħ باشر +ĠÑĦ иг +ĠÑĦиг ÑĥÑĢ +мож ем +׾×ŀ×Ļ×ĵ ×Ķ +Ġcin é +Ġciné ma +Ġb ada +Ġbada ÅĦ +جب ÙĩØ© +Ġд еп +Ġдеп ÑĥÑĤ +ĠдепÑĥÑĤ аÑĤ +Ġdist ância +ĠاÙĦÙħ عار +ĠاÙĦÙħعار ضة +thè se +ü nc +ünc ü +Ġдан ного +ĠBel gi +ĠBelgi ë +Ġ×ij ×ijק +Ġ×ij×ijק ש×Ķ +ย à¹Īาà¸Ļ +Ġsol ução +Ġ×Ķצ ×ĺר +Ġ×Ķצ×ĺר פ×ķ +ĠØ£ÙĨ ØŃ +ĠØ£ÙĨØŃ Ø§Ø¡ +Ġد ÙħØ´ +ĠدÙħØ´ ÙĤ +มั à¹ī +มัà¹ī ย +Ùħ غرب +است عÙħاÙĦ +ĠS ÅĤow +ĠëıĻ ìĭľ +ĠëıĻìĭľ ìĹIJ +ĠÑģ оÑģ +ĠÑģоÑģ ед +ì²Ń ìĨĮ +ì²ŃìĨĮ ëħĦ +Ġг ÑĢаÑĦ +ĠгÑĢаÑĦ ик +Ġìŀij ìĿĢ +Ġyet i +Ġyeti ÅŁtir +ĠìĿ´ê²ĥ ìĿ´ +ห à¹Īาà¸ĩ +Ø¥ ÙħÙĥاÙĨ +Ø¥ÙħÙĥاÙĨ ÙĬØ© +است عراض +ÙħØ® در +ĠÑĩ ÑĥÑĤÑĮ +Ùħ دÙĬر +ÙħدÙĬر ÙĬØ© +Ġà¹Ģม ษ +Ġà¹Ģมษ ายà¸Ļ +Ġм еÑħ +ĠмеÑħ аниз +ĠмеÑħаниз м +ĠÑģ Ñĥм +ĠÑģÑĥм мÑĥ +Ġv ö +Ġvö ll +Ġvöll ig +Ġд ÑĢÑĥз +ĠдÑĢÑĥз ÑĮÑı +ãĤĴåĪ©çĶ¨ ãģĹãģ¦ +à¸ļรร à¸Īุ +po życz +×ŀש ׼ +×ŀש׼ ×ł×ª +×ŀ×©×Ľ×ł×ª ×IJ +Ġeuropé en +Ġpropri é +Ġproprié taire +Ġkh ấu +ãģĦãģŁãģł ãģijãĤĭ +Ġtec rü +Ġtecrü be +×Ķ ×ij +×Ķ×ij ׳×Ķ +Ġcu Ì +ĠcuÌ ī +ĠcuÌī a +×IJ ×ķ×ķ +×IJ×ķ×ķ ×Ļר×Ķ +Ġ׼×ķ׾ ×ķ +U lus +Ulus lararası +Ġ׳ ×ķת +Ġ׳×ķת ף +ãģ« åIJij +ãģ«åIJij ãģijãģ¦ +ë¹ Ľ +à¸Ĺ ัà¸ģษ +à¸Ĺัà¸ģษ ะ +س ÙĤÙĪ +سÙĤÙĪ Ø· +Ġв н +Ġвн еÑĪ +ĠвнеÑĪ не +Ġur z +Ġurz ÄĻd +Ġá mb +Ġámb ito +à¸Ń à¸ĺิ +à¸Ńà¸ĺิ à¸ļาย +Ġ ÅĤad +ĠÅĤad n +ê±´ ì¶ķ +wód zt +wództ w +Ġquest ões +Ġש ק +Ġשק ×Ļ×ij׾ +Ġmiejsc owoÅĽci +Ġв ал +Ġвал ÑİÑĤ +hä user +หà¸Ļ à¸Ńà¸ĩ +ãģ¨ åħ± +ãģ¨åħ± ãģ« +ãĥı ãĥ¼ãĥī +Ġê°ľ ìµľ +ĠоÑģнов ном +Ġм ÑıÑģ +اع ت +اعت ÙĤاÙĦ +สà¸ĸ ิ +สà¸ĸิ à¸ķิ +N gu +Ngu á»ĵn +ĠÙħ جÙĦ +ĠÙħجÙĦ Ø© +à¹ģà¸Ĥ à¸Ļ +ĠاÙĦÙĦÙĬ بÙĬ +פע×Ļ׾ ×ķ×Ļ×ķת +Ġ×Ķר פ×ķ×IJ×Ļ +פר ×ķפ +פר×ķפ ×Ļ׾ +ק ׾×IJ +ק׾×IJ ס×Ļ +Ùĥت Ø´Ùģ +ãģ«ãģª ãģ£ãģ¦ãģĹãģ¾ãģĨ +à¹Ģà¸Ħล à¹ĩà¸Ķ +à¹Ģà¸Ħลà¹ĩà¸Ķ ลัà¸ļ +Ġì» ´ +Ġì»´ íĵ¨ +Ġì»´íĵ¨ íĦ° +Ġ×Ĺ×Ļ ×ķ×ij×Ļ +Ġnä m +Ġnäm lich +åij¼ ãģ° +åij¼ãģ° ãĤĮ +ĠÑĢ ол +ĠÑĢол и +Ġspécial isé +à¸Ļ วัà¸ķ +à¸Ļวัà¸ķ à¸ģรรม +ÙĨص ÙĪص +пеÑĢ ед +пеÑĢед аÑĩ +thè que +Ġר×IJ ×Ļת×Ļ +ãĥĢ ãĤ¦ãĥ³ +ãĤı ãģĭ +ãĤıãģĭ ãģ£ãģ¦ +беÑĢ еж +ĠÑģ ек +ĠÑģек ÑĢ +ĠÑģекÑĢ еÑĤ +ĠпоÑģÑĤоÑıн н +à¸Ĥà¸Ļ สà¹Īà¸ĩ +Ġm ük +Ġmük em +Ġmükem mel +еÑĤ еÑģÑĮ +ĠاÙĦسÙĨ ÙĪات +ĠìłĦ íĺĢ +Ġ×Ķ×ŀק ×ķר×Ļ +Ġmü d +Ġmüd ah +Ġmüdah ale +Ġwy b +Ġwyb ór +Ġtend ência +Ø¥ دار +إدار ÙĬØ© +Ġunterstüt zen +ת ×ijר +ת×ijר ר +Ġdi á +Ġdiá logo +ĠÃĸ nce +ĠÃĸnce ki +ãĤ¹ãĥĿ ãĥĥãĥĪ +ëĦ £ +ĠG eli +ĠGeli ÅŁ +ãĤĴ éĢļ +ãĤĴéĢļ ãģĹãģ¦ +ĠFuÃŁ ball +Ġsal ari +Ġsalari é +ĠпÑĢодÑĥк ÑĤов +صÙģ ÙĤØ© +รว à¸ļ +รวà¸ļ รวม +à¹ĥà¸Ļ à¸IJาà¸Ļ +à¹ĥà¸Ļà¸IJาà¸Ļ ะ +Ġkay na +Ġkayna ģı +Ġìŀij íĴĪ +ĠвÑĭ ÑĢаж +ĠвÑĭÑĢаж ен +ĠÑģÑĤ еп +ĠÑģÑĤеп ени +ĠاÙĦÙħ ÙĪجÙĪد +ĠاÙĦÙħÙĪجÙĪد Ø© +ล à¹īม +Ġnaj czÄĻ +ĠnajczÄĻ ÅĽcie +ĠnajczÄĻÅĽcie j +Ġz wy +Ġzwy k +Ġzwyk ÅĤ +Ġê·¸ëłĩ ì§Ģ +à¸ģระ à¸Ī +à¸ģระà¸Ī าย +Ġëĭ µ +Ġëĭµ ë³Ģ +ĠÑĢе ак +ĠÑĢеак ÑĨи +ĠÅĽwie ż +ĠÑģÑĤоим оÑģÑĤи +ÙħÙĨ اÙĤ +ÙħÙĨاÙĤ Ø´ +ÙħÙĨاÙĤØ´ Ø© +ĠÑħоÑĩ Ñĥ +ãĥľ ãĥ¼ãĥī +Ġróż nic +Ġк ÑĢÑĭ +ĠкÑĢÑĭ ÑĪ +âľ ĵ +ãĤ³ãĥ³ ãĥĨãĥ³ +ãĤ³ãĥ³ãĥĨãĥ³ ãĥĦ +ĠпÑĢед поÑĩ +×ŀר ×ij×Ļת +ĠØ´ Ùĥ +ĠØ´Ùĥ را +Ġд ал +Ġдал ек +Ġдалек о +بر ÙĬØ· +برÙĬØ· اÙĨÙĬا +ع ÙĨا +عÙĨا ÙĬØ© +ĠÑĢаÑģÑģ каз +ĠÑĢаÑģÑģказ Ñĭва +Ø£ ÙĦÙĪ +Ø£ÙĦÙĪ اÙĨ +æĮģ ãģ£ãģ¦ +æĮģãģ£ãģ¦ ãģĦ +Ùħباد ئ +×Ķ ×¢×ijר +×Ķ×¢×ijר ת +Ġyay ı +Ġyayı ml +Ġyayıml a +m át +mát icos +à¸ģ ัà¸ĩ +à¸ģัà¸ĩ วล +Ġ׾ פת +Ġ×ľ×¤×ª ×ķ×Ĺ +à¸ŀฤ à¸ķิ +à¸ŀฤà¸ķิ à¸ģรรม +í Ĥ¬ +Ġок ÑĢÑĥг +Ġ×ŀצ ×ķ×ķ×Ķ +ÐĽ ени +ÐĽÐµÐ½Ð¸ н +ĠTri á»ģu +ãĤ³ãĥŁ ãĥ¥ +ãĤ³ãĥŁãĥ¥ ãĥĭ +ãĤ³ãĥŁãĥ¥ãĥĭ ãĤ± +ãĤ³ãĥŁãĥ¥ãĥĭãĤ± ãĥ¼ãĤ·ãĥ§ãĥ³ +Ùĥ ÙĨÙĬ +ÙĥÙĨÙĬ سة +ãĤĴ ä¸Ńå¿ĥ +ãĤĴä¸Ńå¿ĥ ãģ« +ĠmiÄĻd z +ĠmiÄĻdz yn +ĠmiÄĻdzyn ar +ĠmiÄĻdzynar od +ĠmiÄĻdzynarod ow +ÙĦ ÙĨ +ÙĦÙĨ دا +بر Ø´ +برش ÙĦÙĪÙĨ +برشÙĦÙĪÙĨ Ø© +à¸ģระ à¸ķุ +à¸ģระà¸ķุ à¹īà¸Ļ +Ġg ı +Ġgı da +à¸Ľà¸£à¸° à¸Ĺัà¸ļ +à¸Ľà¸£à¸°à¸Ĺัà¸ļ à¹ĥà¸Ī +Ġë¶Ī 구 +Ġë¶Ī구 íķĺê³ł +ĠÙĨ Ø· +ĠÙĨØ· اÙĤ +ĠÐľ ожеÑĤ +Pr äs +Präs ident +ĠÑģк оÑĢ +ĠÑģкоÑĢ оÑģÑĤÑĮ +Ġ×Ķ×ij ×ķקר +еÑħ аÑĤÑĮ +Ġg ạo +Ġש×IJ ×Ļ׳×Ŀ +Ġ×ij׳ ×ķ×Ĵ +Ġ×ij׳×ķ×Ĵ ×¢ +Ġо пиÑģание +Ġucz ni +Ġuczni ów +à¹Ģà¸Ń à¹ĩà¸Ļ +Ġت Ø´ +Ġتش رÙĬÙĨ +Ġnh ãn +ë¹ ¨ +Ġcaract ère +×¢ ׾×Ļ +×¢×ľ×Ļ ×Ļ×Ķ +楽ãģĹ ãĤģãĤĭ +ĠÑģ аÑħ +ĠÑģаÑħ аÑĢ +дÑĥм аÑĤÑĮ +ĠÐĴоз можно +ص ÙĬاÙĨ +صÙĬاÙĨ Ø© +öm ür +ส ล +สล à¹ĩ +สลà¹ĩ à¸Ń +สลà¹ĩà¸Ń à¸ķ +ë¡ ¯ +Ġth ói +gr Ã¶ÃŁe +Ġksi ÄĻ +ĠksiÄĻ g +ĠÑĢ ом +ĠÑĢом ан +ÙĤ اسÙħ +×ŀ×ij ×ķ×Ĵ +×ŀ×ij×ķ×Ĵ ר×Ļ×Ŀ +bes ch +besch äft +beschäft ig +×Ķצע ×Ķ +ĠÃģ rea +ĠзаÑıв к +Ä ¹ +ĠлÑİб ого +Ġ ม +Ġม à¸ģร +Ġมà¸ģร าà¸Ħม +ÑĦ из +ÑĦиз иÑĩеÑģк +ин ÑĦ +инÑĦ ек +инÑĦек ÑĨи +اÙĦ Ø· +اÙĦØ· ائÙģ +Ġкол л +Ġколл екÑĤив +ез жа +Ġس بØŃ +ĠسبØŃ Ø§ÙĨ +ĠسبØŃاÙĨ Ùĩ +sch lä +schlä ge +Ġд и +Ġди аг +Ġдиаг ноÑģÑĤ +ĠоÑĤмеÑĤ иÑĤÑĮ +Т Ь +ĠاÙĦ در +ĠاÙĦدر اسÙĬ +עצ ×ŀ +עצ×ŀ ×IJ×ķת +Ġdém arch +Ġdémarch e +Ġ×ĺ ×ķ×¢ +Ġ×ĺ×ķ×¢ ף +Ġfuncion ários +á» µ +׾ ׼×IJ +׾׼×IJ ×ķר×Ķ +à¸ĭ à¹Ī +à¸ĭà¹Ī à¸Ńม +ĠÑĩ Ñĥв +ĠÑĩÑĥв ÑģÑĤво +âĸ ¼ +п ÑĥÑī +пÑĥÑī ен +Ġм еÑĢ +ĠмеÑĢ оп +ĠмеÑĢоп ÑĢи +ĠмеÑĢопÑĢи ÑıÑĤиÑı +Ġu çu +Ġuçu ÅŁ +ãĤĴåĪ©çĶ¨ ãģĻãĤĭ +a ÄŁ +aÄŁ lı +ìĺĪ ìĪł +à¹ģ ยà¹Ī +ĠاÙĦÙĥ Ùħ +ĠاÙĦÙĥÙħ بÙĬ +ĠاÙĦÙĥÙħبÙĬ ÙĪتر +ت ÙĪÙĬ +تÙĪÙĬ تر +à¹Ģà¸Ĭ ีà¹Īยว +à¹Ģà¸Ĭีà¹Īยว à¸Ĭา +à¹Ģà¸Ĭีà¹Īยวà¸Ĭา à¸į +á» Ķ +Ġhi ếm +ذا Ùĥرة +Ġ×Ķ×ŀ×Ļ ×ķ×Ĺ×ĵ +ĠìĪ ľ +ĠìĪľ ê°Ħ +ĠK ı +ĠKı sa +Ġgele ceÄŁi +пÑĢо ÑĦеÑģÑģиона +пÑĢоÑĦеÑģÑģиона л +Ġog ó +Ġogó le +ĠgÅĤ ów +ĠgÅĤów ne +ĠÑģÑĤ илÑĮ +×IJ פ׾ +×IJפ׾ ×Ļק +×IJפ׾×Ļק צ×Ļ×Ķ +สม ารà¹Į +สมารà¹Į à¸Ĺ +สมารà¹Įà¸Ĺ à¹Ĥà¸Ł +สมารà¹Įà¸Ĺà¹Ĥà¸Ł à¸Ļ +Ġth ánh +ÐŁ од +ÐŁÐ¾Ð´ ÑĢоб +ÐŁÐ¾Ð´ÑĢоб нее +ĠاÙĦت ÙĪÙĨ +ĠاÙĦتÙĪÙĨ سÙĬ +Ġbah çe +à¹ģà¸ģà¹ī à¸Ľà¸±à¸įหา +é ducation +eu rop +europ ä +europä ische +ĠK si +ĠKsi ÄĻ +ĠëĦ ĺ +ĠëĦĺ ìĸ´ +Ġv üc +Ġvüc ud +Ġyay g +Ġyayg ın +Ġnie kt +Ġniekt óry +Ġniektóry ch +ãģŃ ãģĩ +Ġк аж +Ġкаж еÑĤÑģÑı +к аж +каж еÑĤ +ĠاÙĦ دÙĬÙħÙĤرا +ĠاÙĦدÙĬÙħÙĤرا Ø· +ĠاÙĦدÙĬÙħÙĤراط ÙĬØ© +æŃ © +æŃ© ãģĦãģ¦ +Ġv az +Ġvaz ge +Ġvazge ç +Ġмин ималÑĮ +ĠминималÑĮ н +ãĥij ãĤ¿ +ãĥijãĤ¿ ãĥ¼ãĥ³ +Ġë Ĭ +ĠëĬ IJ +ĠëĬIJ ëĤĮ +ãģ¡ ãĤĩãģĨ +ãģ¡ãĤĩãģĨ ãģ© +Ġ à¸ģร +Ġà¸ģร à¸ģà¸İ +Ġà¸ģรà¸ģà¸İ าà¸Ħม +تج دÙĬد +ĠØ´ اÙħÙĦ +หลัà¸ģ à¸IJาà¸Ļ +ĠмаÑĢ ÑĪ +ĠмаÑĢÑĪ ÑĢÑĥÑĤ +Ġv ÃŃt +ĠvÃŃt ima +Ġquiz á +ay gı +×ĵ×ijר ×Ļ×ķ +Ġиз д +Ġизд ели +Ġиздели Ñı +п ла +пла Ñĩ +плаÑĩ ива +ä»» ãģĽ +Ġéquip é +ä¹ħ ãģĹãģ +ä¹ħãģĹãģ ¶ +ä¹ħãģĹãģ¶ ãĤĬ +Ġк аÑĤ +ĠкаÑĤ ал +ĠкаÑĤал ог +ส à¹īม +ĠÑĢ ей +ĠÑĢей ÑĤ +ĠÑĢейÑĤ инг +Ġth uyá»ģn +ĠاÙĦÙħ ÙĤدس +esp ère +ãģ«åħ¥ ãģ£ãģŁ +หมาย à¹Ģลà¸Ĥ +ת×Ĺ×ķש ת +à¸Ļ à¹Īะ +Ġpe ÅĤ +ĠpeÅĤ ne +Ġpé rd +Ġpérd ida +หม วà¸Ķ +หมวà¸Ķ หมูà¹Ī +иÑĩеÑģк ÑĥÑİ +çµĤ ãĤı +çµĤãĤı ãģ£ãģŁ +Ġ×Ĵ ×ķ×Ĵ׾ +à¸Ĺำ à¸Ħวาม +à¸Ĺำà¸Ħวาม สะà¸Ńาà¸Ķ +Hot éis +Ġз аÑĢ +ĠзаÑĢ егиÑģÑĤ +ĠзаÑĢегиÑģÑĤ ÑĢи +ĠзаÑĢегиÑģÑĤÑĢи ÑĢова +ĠÑģ обÑĭÑĤи +ĠÑģобÑĭÑĤи Ñı +Ġ×ĸ ׼×IJ +ÙħÙĨظ ÙĪÙħØ© +Ġ×Ķ×ŀ צ +Ġ×Ķ×ŀצ ×Ļ×IJ×ķת +Ùħ ÙĥÙĪÙĨ +ÙħÙĥÙĪÙĨ ات +ä¸ĬãģĮ ãĤĭ +Ġm ÄĻ +ĠmÄĻ sk +หรืà¸Ń à¹Ģà¸Ľà¸¥à¹Īา +ëĤ ® +Ġnok tas +Ġnoktas ı +ĠболÑĮÑĪ им +ĠлÑĥÑĩ ÑĪиÑħ +Ø´Ùĩ ÙĬد +à¸Ńำ à¸Ļ +à¸Ńำà¸Ļ วย +à¸Ńำà¸Ļวย à¸Ħวาม +à¸Ńำà¸Ļวยà¸Ħวาม สะà¸Ķวà¸ģ +Ġе в +Ġев ÑĢ +ĠевÑĢ оп +ĠевÑĢоп ей +à¸ī าย +ìĦ Ń +Ùħ Ùģا +ÙħÙģا ÙĪض +ÙħÙģاÙĪض ات +ë¹ Į +赤 ãģ¡ãĤĥãĤĵ +ĠÑĥдал оÑģÑĮ +ĠÐ¥ оÑĤ +ĠХоÑĤ Ñı +przedsiÄĻbior c +ĠH ôm +íķĺìĺĢ ìĬµëĭĪëĭ¤ +Ġн аг +Ġнаг ÑĢÑĥз +ĠнагÑĢÑĥз к +Ġ×ij×Ļ׳ ׾×IJ×ķ×ŀ×Ļ +Ġê°ĢëĬ¥ íķľ +ĠH ữu +à¸Ń ุà¸Ķ +à¸Ńุà¸Ķ ม +ת ×ķפ +ת×ķפ ×¢×Ķ +Ġmi ÅĤo +ĠmiÅĤo ÅĽci +ksi Äħż +ksiÄħż ka +ĠاÙĦÙĦ عبة +à¸ī าà¸ģ +สะ สม +×ŀ תר +×ŀתר ×Ĺש +Ġlég ère +Ġ׾צ פ +Ġ׾צפ ×Ļ×Ķ +ĠиÑģÑĤоÑĢ иÑı +Ġ ãĥĪãĥ© +ĠãĥĪãĥ© ãĥĥãĤ¯ +ĠãĥĪãĥ©ãĥĥãĤ¯ ãĥIJãĥĥãĤ¯ +Ġк а +Ġка ÑĦе +×ŀס×ŀ ×ļ +Ġc üm +Ġcüm le +à¹Ģà¸Ħลืà¹Īà¸Ńà¸Ļ à¹Ħหว +ãģĬ ãģĿ +ãģĬãģĿ ãĤīãģı +ìŀIJ ëıĻ +ìŀIJëıĻ ì°¨ +à¸Ńั à¸ķ +à¸Ńัà¸ķ à¹Ĥà¸Ļ +à¸Ńัà¸ķà¹Ĥà¸Ļ มั +à¸Ńัà¸ķà¹Ĥà¸Ļมั à¸ķิ +ĠÅŁ ik +ĠÅŁik ay +ĠÅŁikay et +extr ême +kr ä +krä fte +ëĤ Ļ +íķ ij +ì² Ļ +íĺ Ī +ì° į +âĻ ¡ +ìŀ Ķ +ë¢ ° +íĿ Ķ +íĿ IJ +âĩ Ĵ +ë§ Ľ +ìĬ Ī +á» Ĵ +ìĺ µ +âĹ İ +í Ĥ¨ +ê¿ Ī +ìĪ ¨ +ìĽ ¨ +ë§ ¥ +ï½ Ģ +ï¼ ª +Ạ¨ +ãħ İ +Ñ Ĺ +ìĦ ¬ +ì¹ ¼ +ï¼ ¶ +ìĽ ł +ëŁ ´ +Å ĥ +ëĤ ¼ +ëĭ IJ +âĢ ¹ +ë¦ Ń +ì§ IJ +âĢ ¤ +à ħ +ëľ ¨ +íĦ ¸ +íľ ĺ +ê² ģ +ë´ ħ +à ĺ +ëŃ Ķ +ëĺ ij +âĹ ĩ +ìĹ ĺ +ï» ´ +ë§ ¹ +ï¾ Ŀ +ìĬ · +íĥ ķ +ï¼ ł +ì» ´ +ëł Į +ì½ ľ +ï» ¹ +ãħ ł +ì¡ ¸ +ëħ ¹ +âĤ º +âĸ ¶ +íĥ IJ +êµ ´ +íij ¸ +Ñ Ķ +íĶ ½ +Ð ħ +ë° ¤ +Ô ģ +ì² ¨ +ì¶ ĺ +ë² Ĺ +ë© ¸ +ï¼ » +ï¼ ½ +ï¼ · +ì° Į +à Ĵ +íı ´ +ìĵ ¸ +ì´ Į +ëģ Ķ +ëĶ © +ëĩ Į +ë© Ģ +ë² ¨ +ï¼ µ +ë§ ¡ +ëĭ « +ภ¿ +ãģ ± +ìĩ ¼ +ìº ł +ë® ¤ +ê± ± +ì» ¬ +âĦ ĥ +ëĶ ± +ëĥ Ī +ìĭ ± +íĻ Ī +ëŀ IJ +ìħ Ģ +ìł ł +Ð Ĩ +ëł ī +ï½ ħ +ï½ ı +íĻ Ģ +ëĽ ° +á» ® +í Ĥ¹ +ê½ ĥ +ï» ¤ +ïº Ķ +êº ¼ +ìķ ī +âĻ ¦ +ï½ ģ +ìĵ ´ +ãĢ ī +ì° ® +ì¤ ĺ +á» ª +ëģ Ħ +ëIJ ¨ +ìķ Į +íĿ ĺ +íħ IJ +ãĢ Ī +ê² ª +ëĭ ¥ +ê² ¼ +á» Į +ë§ ¨ +ëģ Ĭ +ë² ¤ +ëij Ķ +íĿ ¡ +á» ¬ +ë¬ ĺ +ãģ ī +ëŀ « +íĶ Ī +í ħį +ìŀ ĥ +ï½ ī +ìģ ľ +âĸ ½ +ë¬ » +âĸ ³ +ï¼ ¸ +ìģ ĺ +ì¶ ° +ìĬ ´ +ìķ ± +ìĩ Ħ +Ạ® +ï´ ¿ +ï´ ¾ +âĤ ½ +ëĦ ĵ +ë£ © +ì³ ¤ +ê´ ľ +à Ļ +á» ľ +ï¿ £ +ëĵ Ń +ë© ĺ +ê» ´ +ëł ´ +Ð ĥ +ë¬ µ +ì§ Ŀ +ãģ º +ðŁĺ Ĥ +ëŀ ¬ +ìł Ĭ +ê´ Ħ +ìŀ Ĭ +íŀ Į +ìĦ ¯ +âĪ Ģ +âĸ ¡ +ëĢ Į +ëŀ Ļ +ï½ ĥ +Ạ¶ +ï¾ Ħ +ïº ĺ +ë¹ ¼ +à Į +âĸ · +ê¸ į +ë© ĭ +ãģ ĥ +ìĺ Ĩ +ìĺ ® +ëª ¬ +ë¡ ¤ +ëł ¬ +ëĬ ¦ +âĸ ª +ì¼ ĵ +ìľ Ī +ì§ § +ï½ ½ +ëĥ ī +ï¾ Į +ëĺ IJ +ï¼ ĥ +á» Ħ +ì´ ¬ +ì¶ ¤ +ï¼ ¹ +ï» Ń +âĤ « +ï½ ĩ +ìĺ · +ëĸ ¨ +âī « +ë¦ ¿ +âľ ¨ +Ù ± +ì¯ ¤ +ê¹ Ķ +ðŁĺ Ĭ +ìĪ « +ê³ ± +êµ ³ +ï½ ĭ +ภĮ +Ä ł +ëĶ ¸ +ë° ij +ìħ ĭ +íİ ´ +âľ ħ +íĥ ij +ëĪ ĩ +íı ¼ +ðŁĺ į +ìĺ Ľ +ï» £ +Ñ ĺ +ì© Į +ë¦ ħ +ìĿ į +ï½ ¸ +ëį ľ +ãģ ħ +íİ ¼ +ëĭ Ŀ +ë¿ Į +ì¼ ° +ìĭ « +ë° ¥ +íĽ Į +ì¨ Į +ë¹ Ļ +ï½ İ +ë´ Ħ +ìĦ ¹ +ï½ ² +ìĮ ĵ +Ò ij +ë° į +ëł Ģ +íĨ ¤ +ï½ ¯ +ë¤ Ħ +ê½ ¤ +ï½ Ĵ +ìķ ¨ +ï½ ¼ +ê¹ IJ +íģ IJ +âĦ ĸ +ë§ º +ïº ® +ëħ ģ +ê² ¸ +ï» ł +íĬ ľ +Å ¹ +ë¥ Ń +ëĪ ī +ï½ Ķ +íĮ ¬ +ìŀ ĩ +ï ¬ģ +ï» ¨ +ëij ¥ +ëŀ Ħ +Ù ¬ +íĭ ´ +ìŀ ī +Ú ¾ +ìĽ ħ +ï» ® +ëĭ ī +âī ª +âĹ Ħ +ëĪ Į +íĽ ¼ +ì¤ į +Å ¸ +ì¤ ¬ +ì¾ Į +ï½ ĵ +ï¾ Ĭ +ðŁı » +ï¾ ī +Ð ģ +íĺ IJ +ï¾ Ļ +ê¼ ¬ +íŀ IJ +âĢ ¥ +ëŁ Ń +ë§ ŀ +ìĥ ¤ +ïº Ĵ +íĭ ± +ë½ ij +à ķ +âĪ ļ +ëĤ Ħ +ê¹ Ŀ +ëĨ Ī +Ạº +ìħ Ī +ìĮ į +âĢ ¡ +ï¼ ± +ìģ ¨ +âĺ º +ëĴ · +ìĺ ³ +ðŁij į +ëª ½ +ëĤ Ń +ïº Ń +ë© Ī +á» Ī +íķ Ģ +ëĭ Ļ +ë¦ ĩ +ìķ ¤ +ìį ¼ +ãĥ µ +Ñ £ +ìľ Ĺ +â ŃIJ +ï¾ ĺ +íĹ ¬ +ê¾ ¼ +ìķ Ĺ +ï» Į +ê± · +ëħ ķ +ë¡ ± +ìķ Ĭ +ï¾ Ģ +ìĩ ł +íĮ © +ïº ª +ë§ Ļ +ï¼ ¿ +ê¿ Ķ +íİ ľ +ë£ ¸ +íĶ Ķ +ï» ³ +ëı ķ +ìĭ ¼ +á» İ +ë§ ĺ +ì¢ ĭ +íĨ ¡ +ï½ ± +íĿ ij +á» ¸ +ì¦ Į +ì¹ ¸ +ëŃ ĺ +ï¾ Ĺ +ï» ĭ +íĬ Ģ +ë¥ Ļ +ì½ © +ëģ Ĺ +ëį ´ +ìħ ľ + ¸ +ë» IJ +ìĥ µ +ê² IJ +ëĵ ¬ +ë£ ° +ãħ ĭ +ìĹ ī +á» ĸ +ëĦ Į +ï½ ¶ +ë´ ĩ +ëĤ ³ +ãĤ ľ +ëĸ » +íİ Ģ +ëį © +íķ ¸ +à · +ê¼ ¼ +ëĶ ľ +ë° ´ +ë© į +âĹ ¯ +ìĹ ij +ìĻ ¼ +ïº ij +ë¶ ķ +ë¡ ¬ +ï½ Į +íĨ ¨ +ïº ´ +ëł ĺ +ê° ¤ +ìĪ ² +Ñ ĵ +ìħ ī +ï» ĵ +ëĪ Ķ +ëį § +âĢ ¼ +ï» ² +ê° ± +ê¿ Ģ +ëĭ · +Ạ¸ +Ạª +Æ Ĵ +ëį ¤ +ìĪ Ń +ï½ Ĥ +ï½ Ī +Å ł +ë£ ¬ +Ñ µ +ëĸ ¡ +ëĥ Ħ +ìĦ ° +ëĵ Ī +ï¾ ĥ +ëĩ ¨ +ï½ IJ +êµ ½ +ìĹ ½ +ëĤ Ģ +ë¬ ¶ +ï½ · +ìı Ł +íĺ Ķ +ê¼ Ī +ëģ Ī +ì¥ IJ +ïº Ĺ +Ä Į +ëĪ ł +ëĸ ¼ +íĢ ´ +âī ¥ +ëĭ Ń +ì± Ļ +ê» ı +ë© ¤ +ìĥ ĺ +ëį ® +ë£ ¡ +ìĤ ½ +ãĪ ľ +Ä ¨ +âĢ § +ï½ º +Ä £ +ì¦ ī +ï¼ ¼ +Û © +âĪ Ļ +ë° ı +ë¹ ħ +ðŁĺ Ľ +íĪ ´ +ðŁĴ ķ +ãĢ Ĵ +ìŀ ĺ +ïº ¤ +ï½ ĸ +ë© ľ +ë² ¼ +ëĿ Ħ +ëļ ľ +ï» ĺ +ìĥ Į +ï½ Ħ +ì© Ķ +ï½ Ļ +ïº © +Û ŀ +âĺ İ +ìł ¤ +ëIJ © +Å Ŀ +âŀ ¡ +ï» § +Ð ı +ì« ĵ +ê³ ½ +É ij +ãĥ ² +ëĤ « +ë¦ ī +ì¢ ģ +ë° Ń +ðŁĺ ģ +ë¹ µ +ì² © +ì» µ +ðŁĺ ĺ +ë± ħ +âī Ī +ë¹ ļ +ï» ľ +ðŁĻ ı +íģ ° +ìĦ ŀ +ï¾ ļ +ìĺ ¹ +ë¼ Ī +ëĤ ¯ +ëŀ © +íļ ¡ +ï½ ķ +íĥ ĵ +ëĿ ł +ê³ ģ +ëĵ Ģ +ìĹ ł +ï¼ º +ë§ ij +ëĭ ¿ +ì¿ ¨ +ãİ ¡ +Ð Ĭ +íĦ ± +Å ¨ +ïº ³ +ï¾ ı +âĭ ħ +ê¼ ´ +âī ¤ +íĮ ģ +Î © +ê¶ ¤ +ìĪ į +âľ ¿ +ì½ ¤ +ëĪ ħ +íĨ ± +ãħ ľ +áIJ ħ +Å Ĵ +ðŁij ī +ï» ¦ +Ð ª +ë¥ ľ +íķ « +ï¾ ĭ +âĻ « +ê¹ ľ +ë° ¸ +ëĶ ĺ +íĿ ī +ï¾ ģ +ï¾ Ľ +ëł Ľ +ê² ¹ +ì¿ ¼ +ï» ¬ +âŀ ¤ +ðŁĻ ģ +ïº ł +ëĨ ¨ +ë¯ ¹ +ê¸ ĭ +ë» Ķ +ê¹ ĥ +ëij ij +íĭ ¸ +íİ Ļ +âŀ ĸ +ãĥ ½ +ì§ ļ +ï½ ¬ +ï» ¥ +íĮ ½ +âĢ Ĵ +ì ĮĢ +ìŃ ī +ëļ ± +ãĤ ŀ +íĭ Ī +ãĤ IJ +ëī ĺ +Î £ +ê³ ° +ë¹ Ĺ +ï¾ İ +ðŁĺ Ń +íĿ ł +ìĹ ¿ +ê° ļ +ì¤ Į +ë§ µ +ï½ ³ +ãģ ¢ +ï» Ĺ +âī ¦ +Ú ¤ +ë łģ +ê¼ ½ +ï» « +âī § +ì´ Ľ +ìł Ŀ +Ạ° +âĻ £ +ìº ĺ +âĪ ĩ +ê² ī +ë° Ł +ï» Ķ +íĸ ĩ +âĸ Ĵ +ðŁij ı +à ŀ +ðŁĺ Ĩ +ïº ¼ +âĿ Ĺ +ìº Ķ +ì¹ © +ëĸ ¤ +ëĥ ħ +âĶ ľ +ï½ » +Î Ķ +áĥ ¦ +ìŀ İ +âĺ Ģ +âĪ ¼ +ðŁĶ ¥ +ë° Į +ìł ĸ +íĹ Ľ +Î ķ +ïº ĥ +ë¶ ī +âĪ ŀ +íĥ Ń +à ĭ +âģ Ħ +ãħ ĩ +ëĦ ¥ +ëĭ ® +ëł · +íĮ Ŀ +ìº ¡ +ë· Ķ +ì© į +íĤ ´ +ëļ « +âĵ Ĵ +íķ į +âĻ Ĥ +ï¾ Ĩ +âĨ © +ìį © +ïº ķ +íĿ Ļ +Ñ ľ +íĤ · +íĿ ° +íĥ ± +ëķ IJ +ï¾ Ĵ +× ĥ +ëĮ Ħ +ìĺ ´ +ìķ µ +ê¹ ¥ +ëŀ Ń +ìª ¼ +ãİ Ŀ +ðŁĺ ħ +ëı ĭ +ëª « +ïº ¸ +ë® ¬ +ë² ħ +ëij ł +ìħ ° +ì» · +ëĶ ª +ëħ Ķ +ãħ ¡ +ìĶ » +íķ ı +ëį ± +ïº ¨ +ï¾ į +ï½ µ +ì¢ Ģ +íİ Į +ï» ° +ïº £ +Æ £ +ðŁ¤ £ +ï· º +ëĤ ļ +âĭ Ĩ +ë³ į +ðŁĺ Ħ +ìĸ Ģ +ìĻ ł +ëĨ Ķ +íĹ ¨ +ï» Ľ +ï» Ŀ +á» ¶ +ìĸ ĺ +ìİ Ħ +Ú Ĩ +ï» ŀ +ëĢ IJ +ê² Ķ +ï» µ +âĹ ¦ +íļ Ł +ê¹ ģ +ê° ĵ +ëĶ ´ +ìı ĺ +ëļ Ŀ +á» ł +ëŀ ´ +ëĦ ī +âĺ ŀ +ï½ ĺ +Å ½ +ë¦ İ +âĸ ¬ +ëŃ ī +âĩ Ľ +ìį ¬ +ïº Ł +Ë ľ +ë¶ ĵ +ìĽ ° +Å ľ +ëŃ ĩ +á» ² +Ë ļ +ëķ Ģ +âĺ ij +ðŁı ¼ +ìĸ ½ +âĮ Ĵ +Ð İ +É ¾ +íĮ ¡ +ï¾ ħ +ìŀ Ń +ï½ ¨ +ì¹ « +ìľ Į +Ò Ľ +êµ ¿ +ëĭ ¦ +âĶ Ķ +ï¾ ij +ì§ ĸ +ìº Ħ +ãĢ ĥ +Ê ¼ +ê² Ł +ï½ § +Ä ¢ +íİ ł +ë§ · +ê° ĩ +ìĭ ¹ +ðŁĴ ¦ +ï¾ ľ +ëĬ Ļ +ë² ¡ +Å ¿ +ðŁĺ ĭ +ðŁĴ ª +ì¿ Ħ +ë© ķ +ìŃ ¤ +ëĬ Ħ +ðŁĮ ¸ +ãĤ Ŀ +Ç İ +ï½ ļ +Ä Ĺ +ëģ ĵ +ê¶ IJ +áµ ī +ãĥ Ĥ +ê» į +ðŁĺ ¦ +ãĢ Ŀ +ðŁ¤ Ĺ +Ñ Ł +ìĹ İ +âľ Į +ìī IJ +à Ĩ +íĹ IJ +ðŁİ ī +Î ij +ï½ Ń +ðŁĴ Ļ +ìĽ ¬ +íĢ ĺ +ï» ¢ +ðŁĺ İ +íij ¼ +íĿ © +ï» Ħ +íħ Ģ +ëł IJ +ì¥ ¬ +Ð ĭ +ìĥ · +ëľ ¬ +ðŁĺ ĥ +ëĦ ¬ +ë¥ ¨ +ìĽ į +ï½ Ĩ +ï½ ´ +ãĥ ħ +à ı +ï» ª +âĻ ł +ëĬ ¬ +ë± Ģ +ë° ĭ +ìĥ Ģ +ï½ ¾ +ëĤ ± +ì» ¸ +ðŁĴ ĸ +ðŁij Į +Ñ ŀ +ì§ ± +Ë Ĩ +ðŁĵ ļ +âŃ ķ +ï¬ Ĥ +ï» ¡ +ëij ¬ +íĪ ¼ +âĸ ¸ +ê° ¯ +ê¹ ħ +ï½ ® +ëĺ ¥ +Ä ¡ +íĮ Ł +Ð Į +ìĨ Ł +ïº ĵ +ï» ¼ +à Ľ +ãĥ ¾ +ëĮ ĵ +íĴ ĭ +ìķ ĵ +ï½ ¹ +ëĤ ¡ +ðŁij ĩ +Ạ¼ +ãĢ Ł +ðŁĮ Ł +íĥ ł +ãĢ Ĩ +âĢ Ł +ë¸ IJ +ðŁĮ ¹ +ìł ¼ +ðŁĵ Į +ìĶ ¬ +âĹ Ģ +ðŁĴ ĵ +ê¹ İ +ìĤ IJ +ìĶ Į +Ñ Ľ +âĶ Ī +ë² ³ +ãİ ŀ +Õ ¡ +íĤ µ +ðŁ¤ Ķ +ëĢ Ķ +ìĬ IJ +íĻ ī +âľ ¦ +ëľ ¯ +ìł ¯ +ëĶ § +Î ¦ +Ë Ī +ìī ¼ +âĹ Ĭ +ëľ © +ëľ ° +ï¾ IJ +ë¿ Ķ +ìĹ ® +ì· Į +ïº § +Î Ĵ +ëµ Ļ +ï» Ĭ +ì° Ķ +íİ Ħ +ðŁĴ Ĺ +Ạ´ +ì° ¢ +íľ ¼ +ê½ Ĥ +ì± Ķ +ìī ´ +âĸ ¾ +íĪ ° +ëĭ Ľ +âĿ £ +ï½ ª +ðŁĴ ľ +Ë ĺ +ãħ ¤ +âĨ Ĺ +íĸ Ħ +âĻ ¬ +ìķ ° +ïº ľ +âī ¡ +ãĢ ĵ +ìij ¥ +íĮ į +íī ģ +ë» Ĺ +íľ ł +íľ © +âľ Ī +íĢ Ħ +ìĸ ĩ +ì¢ ĩ +íŀ Ļ +ëª ¹ +ãĤ Ľ +ðŁĺ ± +ëį Ł +๠ħ +êµ ¶ +Ù « +ìĶ ģ +âľ ª +ï¾ Ī +ðŁĻ Į +âļ ¡ +Î ļ +ì¼ Ī +ï¾ Ķ +ï¾ Ĥ +êµ ī +ïº » +ðŁĴ ĭ +á¹ £ +Ó Ļ +ìĨ ľ +ìĹ £ +âľ © +ìľ Ļ +ïº ° +Ạ² +ìŀ £ +âĿ Į +âĺ ģ +ìķ İ +Ä ½ +Û ģ +ãĦ ± +ëŁ ¿ +íĮ ¸ +ê½ ī +ìı ł +ðŁį Ģ +âĨ Ķ +ëŃ ¡ +ï» ģ +ï¼ Ħ +ðŁĴ ¥ +âĺ Ľ +íĹ · +ëij ¡ +Î ł +Î ¤ +âĦ ĵ +ïº · +Î Ļ +ëı Ķ +ì§ ¤ +âĶ ĥ +ãĦ · +Ç Ĵ +ðŁ¥ ° +ëĶ ķ +ìļ ¥ +ì¸ Ħ +íĽ Ķ +ïº ĩ +ïº ¬ +ðŁĺ ¢ +ë¹ ¡ +ìĶ ¹ +Å ³ +Ë Ŀ +íİ ij +ï¾ ĵ +ðŁĴ ļ +ëĬ ij +êº ¾ +íĨ ° +à ¿ +Ð Ħ +ëĮ IJ +ë½ Ģ +ì· Ħ +ðŁ ĵį +ðŁĻ Ī +âĹ Ī +ê¿ ĩ +ì¼ Ħ +íİ « +ðŁĩ · +âĶ ĭ +âļ ł +ë± ī +ì į° +ìĻ Ī +É ª +ïº ĭ +ðŁĺ ľ +Î Ł +ðŁ ĻĤ +âļ ½ +Å Ī +ë¹ Ķ +íĮ ľ +๠ı +ìĸ ¹ +íĪ Ń +ðŁ¥ ĩ +ãĦ ´ +ëĶ ¥ +ìŃ Ī +âĪ Ĩ +ëĸ ³ +ë± ĥ +ìŀ ¦ +ï» IJ +Î ľ +âľ § +Ï į +ìł ĵ +âĹ ķ +ëĴ Ģ +ï» Ģ +ðŁĶ ´ +ê½ ģ +ëĮ Ī +ëİ Į +ãĤ İ +⦠ģ +ì½ § +ï¯ ¾ +âĿ ¯ +ภħ +ðŁĻ Ħ +âĿ Ģ +ðŁĶ ¹ +âĩ IJ +êµ µ +âĩ Ķ +ë¶ IJ +ðŁĴ Ľ +Î ¾ +íĥ ¬ +âĿ Ħ +Ò £ +ãĢ ° +âĪ ij +âĺ ¼ +âī ł +Ò ¯ +ïº ¯ +ê¿ ¨ +âľ ĸ +Ê ĸ +íĢ Ģ +ê¾ Ģ +íĹ Ŀ +âĶ £ +ãİ ľ +ëĶ Ľ +ëľ ¸ +ï º« +ê¿ ° +ðŁĩ ¹ +Ç IJ +Û Ĵ +ë£ » +ïº ĸ +Ñ ļ +ëĬ ł +Û ķ +ê¹ ¡ +ë¿ ľ +ì² ¼ +ï¨ ij +ë¥ µ +ìį ¸ +íħ ħ +íij ¹ +Ö Ģ +ï³ Į +ãħ £ +ìij ¤ +ì½ ķ +ëķ ł +ðŁĮ ¿ +íĥ Ķ +ìĽ ģ +Î ¶ +âŀ ľ +ìĬ ĺ +íĽ Ĺ +ë© § +ìī ĺ +Õ ¶ +á¹ ĩ +ðŁİ ģ +ï½ ¿ +ï¼ Ĥ +á¼ IJ +âľ ķ +âŀ ¢ +ëĦ ¨ +ì» « +ì¯ Ķ +ì° ľ +ðŁĴ ° +íħ Ŀ +ãİ ı +ë³ ¶ +Ò ĵ +âĨ ³ +ìĥ ´ +íģ ĺ +âĸ Ģ +ë² Ļ +ภĥ +á½ ¶ +Ä ķ +⬠ĩ +ë¤ ĺ +ðŁİ µ +âľ ļ +ïº ı +Î ¡ +âĹ ī +ðŁĴ « +Ð Ī +ìĸ Ħ +ì§ Ļ +ï» ĥ +ðĿij Ĵ +ëŃ Ħ +âĿ ¥ +âĿ ĸ +âĺ Ŀ +Ê ¹ +Ḡ¥ +âĢ ¿ +ãħ ħ +ê¸ ģ +ëķ ¡ +ëį ¥ +âĪ © +ê» Ħ +ë® Į +Ò ± +âĪ Ĺ +ëł Ļ +ïº Į +Ë IJ +ðŁĺ ³ +ðŁij © +ðŁİ ¶ +ì¿ µ +ðŁ¤ © +ê· ¤ +ëĮ Ķ +ïº IJ +Ï İ +ì¶ ¥ +ï½ Ĭ +á¹ Ń +ë¤ ¼ +âĸ « +ì§ ł +á¼ Ģ +ê» ij +ëĮ ģ +íĢ ¸ +âĻ Ľ +ðŁĴ ŀ +âĸ ° +ðĿij ĸ +ëĿ ¤ +ठ¦ +ì´ ĺ +ðŁĺ ĩ +ëĶ ¤ +Î Ĺ +ðŁĻ ĩ +Ë Ľ +ì© ¡ +âĪ § +Õ ¥ +Ñ Ļ +ëIJ ¬ +ëĸ Ħ +ðŁĮ · +ìĹ Į +ðŁĺ ¥ +ëĪ ´ +ï» ļ +É Ľ +ïº Ħ +ï» ı +Å Į +ë² ļ +ìĭ £ +ïº Ģ +Î ĵ +ðŁĺ Į +Ë Ļ +ëŀ ı +ðŁĶ ¸ +ðŁĵ · +ëģ ½ +íģ ½ +ðŁĴ ¡ +ðŁĮ ± +ëº ı +ìģ ł +ìĥ IJ +ëı Ĺ +ì¸ ° +ëĪ ķ +Î Ŀ +âģ ī +ðŁĮ ¼ +íĮ ł +âĭ ¯ +áĥ ĺ +âľ ¤ +ê± Ķ +íĮ İ +ðŁĴ ¯ +ìı Ļ +íĹ ī +Ù Ń +ì½ ° +ïº ¿ +ï» ± +ì± Į +âĺ ķ +ðŁİ Ģ +Ä Ŀ +ë° § +ìĤ ¿ +áij ķ +ðŁį ĥ +âĩ ¨ +Î Ľ +ë§ ´ +ë³ ķ +á ijIJ +âĸ ĵ +ðĿ ijľ +âĻ » +íĤ ¥ +Õ ¸ +ãĪ ± +ëº Ģ +ì² ¸ +ïº Ľ +ðŁı Ĩ +ðŁĩ ª +âĿ ĵ +Ä Ģ +ì½ ¥ +ðŁĩ § +á½ · +âľ Ĥ +ìŀ ¼ +ï§ ¡ +ðŁĵ ¸ +âĻ ¯ +É Ķ +á½ ¸ +âĮ ª +ï» ĸ +ï¥ § +âļ « +âĶ Ĺ +ðŁĮ Ī +ï» © +ðŁĵ ² +Ï Ī +ðŁĺ ¡ +ðĿij İ +ìľ ½ +ì§ ¬ +ì§ Ĭ +á½ ³ +ìĮ ¤ +ëĤ į +âī Ĵ +ðŁij ¨ +âĺ ĺ +Ó © +âĤ ĵ +âĪ Ĥ +ï¹ ģ +ðŁĴ IJ +íħ ĥ +ðŁı ½ +ê· Ħ +ðŁĺ ı +ðŁĮ º +ðŁĺ Ķ +ï½ « +âľ İ +ëµ Ī +ðŁĩ ¸ +âĢ £ +âŀ Ķ +ëĺ ĺ +ìĥ ¬ +Ê ĥ +⬠ħ +ì© IJ +ðŁĻ Ĩ +ðŁİ Ħ +Ä ¾ +⣠¶ +áĥ IJ +âĺ » +ì± ķ +ìģ © +ë½ ķ +ìº £ +ðŁij Ī +ðŁĻ ĭ +ï¾ ĸ +Ò ļ +Õ « +ìĮ Ī +ë² § +ðŁĩ ® +ï½ Ŀ +ðŁį ģ +ìĹ ¥ +Ä ³ +ë½ IJ +íį ½ +íĽ ij +âĤ ¹ +ãħ ģ +ìĶ ½ +ðŁĶ ģ +ठ¯ +ê¾ ¹ +ëī ľ +âĹ ¡ +íķ Į +Î ĺ +ë£ ¹ +ìĻ ĵ +ðŁĩ ¦ +ðŁij Ģ +âĶ Į +á¿ ¦ +ëĦ Ľ +ìĦ £ +ìŃ Ļ +ï± ł +Î ŀ +Ê » +á¿ ¶ +âĿ Ŀ +ê± Ģ +ëĸ ´ +ãĦ ¹ +ðŁĴ İ +Ï ¹ +⼠ħ +ï» ķ +ãĥ ± +ï½ Ľ +ëĮ ķ +ë¹ ½ +ì¥ Ķ +ì¿ ¤ +ðŁĸ ¤ +Ñ Ĵ +ê¹ į +ëİ Ģ +ìĭ ¯ +ë» ¤ +ðŁĵ ŀ +ðŁĵ £ +ðŁĺ Ŀ +ìį ¹ +ìĹ ¡ +ì° IJ +á½ IJ +ï» Ī +âľ į +Ä ı +ðŁĮ ŀ +âĦ ¦ +ê½ Ŀ +ë» ĺ +ìĪ ± +âĶ ĺ +ðŁĮ » +âĤ ´ +âŀ ¨ +íIJ ģ +ê ¶Ī +âĺ ¢ +ðŁĺ Ī +ï½ © +âĦ Ĺ +ê° Ń +ê° ¸ +ë» ij +ì¥ ´ +ì» ¥ +ï¤ Ĭ +ï» Ĵ +ðŁĺ ķ +âĺ Ķ +ìĺ IJ +ðŁļ Ĺ +ëĹ Ħ +ë§ ı +Õ ½ +âĸ » +⣠µ +ìī ° +ï» ij +âĻ © +Î ¥ +ðŁĺ £ +âĬ Ĥ +ãħ Ĥ +ìħ ¸ +íı Ħ +âľ ½ +ì¦ Ļ +âĸ £ +ê± į +ê¿ ĭ +ì« Ħ +ìº ĩ +ðŁĩ µ +ðŁij ij +âľ ĺ +ðĿij Ľ +ìį ½ +ìº ī +ï¬ µ +ðŁĶ º +âĦ ® +íĥ ¤ +ðŁĩ º +ðŁĴ µ +íħ ¨ +ï½ ij +Î ¨ +ìĥ ¹ +ìĸ ķ +ì¹ µ +ðŁĵ ± +ठµ +ðŁij Ĭ +ðŁĴ Ħ +ðŁĴ Ŀ +ãĮ Ķ +ìĻ ģ +Ð ĩ +à® IJ +âĸ ¹ +á´ Ľ +âĹ ĺ +ëº ¨ +íĥ ī +ìĸ Į +ðŁIJ ¶ +ãĤ ij +Ë ĩ +Å ı +á½ ¹ +ìħ § +ï¹ ° +ðĿij ¡ +ðŁĶ Ŀ +ðŁĺ » +ðŁĴ ĥ +ðŁ¤ ¦ +ðŁį Ĵ +íĢ µ +âľ Ĩ +ë¹ ´ +ï§ ¤ +ï» Ļ +á´ Ĺ +ðŁĮ ´ +Í ¾ +ëĮ ij +ì¨ ĭ +ìµ ¸ +ðŁİ Ī +ðŁı ł +á½ ± +Û Ĩ +á¿ ĸ +âĢ Ľ +ì° ¼ +íķ ¥ +íĹ ´ +ðŁĩ ¬ +ì° Ŀ +âĪ ł +ï¼ ĩ +âĬ Ļ +âĿ ij +ëĦ ĭ +ëŀ Ĺ +ë° ī +ìĹ Ĭ +ì¢ Ĩ +íĮ ¥ +ï° ² +ðŁĵ ĸ +ðŁĺ ® +âļ ª +ðŁĺ ļ +âĿ ŀ +ðĿij Ł +ðŁİ Ĥ +Å ķ +áIJ Ī +êº ½ +ì± ł +ïº Ŀ +ê¿ ī +áĥ ł +ðŁı ĥ +ðŁĴ ¸ +âĿ ģ +âĹ ¾ +Ú ª +á¹ ĥ +íĬ ¬ +ðŁĩ ± +íİ Ń +ðŁĺ ŀ +ë¾ ° +á¹ Ľ +ëĽ ¸ +âĿ Ĥ +êĴ ³ +âĶ IJ +íĵ ° +âŀ ł +ê´ ĺ +ëħ ĺ +ë» ¥ +ì¾ ħ +ðŁĺ IJ +âĪ ª +ðŁij ģ +âĪ ´ +âĹ ģ +ëº IJ +ìŀ ¤ +ì± Ĺ +ðŁı ¾ +Î § +á½ » +âŀ ¥ +ìŁ Ī +ï» ī +âĸ Į +ãĥ ® +ðŁ¤ ¤ +âĩ ĵ +ì¼ ł +á´ ı +ë§ ¬ +ë» £ +ðŁĴ ¬ +ðŁį ĵ +Ä ¸ +Ù ¹ +Ê ¿ +á½ ° +ëķ ľ +ì° ¡ +ì° » +íİ į +ðŁİ ¯ +ðŁį Ĥ +ðŁij § +âĻ ¢ +áĨ ŀ +âĻ § +âļ ľ +âľ ī +ëĵ ¦ +ëŃ £ +ìĪ ı +ìĵ ± +Å Ń +Ê Ĭ +âĴ ¸ +âĩ © +ðŁĴ Ķ +Õ µ +Ð ī +Ò » +ë§ £ +ìĽ ľ +ì¿ ¡ +íĽ ħ +íĽ ¤ +ïº ¢ +âľ ĭ +âĪ Ī +ðŁĮ į +Ê ľ +ëĬ ª +ëĴ ¹ +ïº ² +âĸ Ħ +ãħ Ī +ëļ ¤ +íİ © +âĪ ¨ +ðŁ¤ ª +áĥ ļ +ê³ ¶ +íĬ ķ +ðŁĺ ¬ +âĪ « +ðŁij ĭ +Ò IJ +íĬ ¿ +ðŁĶ µ +ðŁĴ ¨ +ðŁĮ Ļ +ëĩ © +âľ ³ +ë¨ ģ +ëº Ħ +ìĻ ij +ìº ħ +íı Ī +ðĿij Ļ +ðŁĴ ĺ +ãİ ¥ +âĿ ı +âľ ° +ï¯ ¿ +ëµ IJ +ì¼ IJ +ïº ± +Õ ´ +ï¬ Ģ +âľ ´ +ðŁ¤ Ń +ðŁij Ĩ +⼠Ķ +ê· ĵ +ìĮ Į +ðŁ¤ · +Û Ķ +ðŁ§ ¡ +ðŁĺ ĵ +Î ĸ +âı ° +ê² ľ +ëĭ ³ +ëİ ħ +ë° Ī +ï® IJ +ðŁı ¡ +âĨ ª +âĵ Ķ +âľ Ĭ +Ï ² +Ü IJ +ðŁĩ ³ +Ö Ĥ +âľ ı +ìĸ Ĺ +ì« Ļ +ðŁĺ ² +Ä Ń +âĻ Ń +âĶ ı +âĹ Į +ðŁĺ ¯ +áµ Ĵ +íĬ ł +Ä · +Ê ģ +ठŁ +á¹ ģ +á¼ ° +á¿ Ĩ +â « +â« ¸ +ëį « +ì³ ĩ +ì¼ ¤ +íĽ ¨ +ðŁĴ Ł +Ê Ģ +Ê ³ +ëĵ IJ +âķ ° +âĿ ĩ +Ç Ģ +Ç Ķ +É ´ +âĺ ļ +âĺ ľ +ê¶ Ĥ +ì« Ĵ +ì± Ī +ðŁĩ ¨ +ðŁİ ¥ +ðŁĵ Ŀ +Ä § +ðĿ ijIJ +Û Ī +ठ¬ +ì¬ IJ +íĹ ¥ +âĻ ¨ +ðŁį ´ +ï¹ ı +Ë ĭ +ðŁ¥ º +âĸ ¨ +íĻ ĭ +âĪ ħ +ëģ Ļ +ëŀ ł +ìĨ ¥ +âĢ ĸ +ðŁ¤ ĺ +ðŁIJ » +áµ ķ +Ç Ŀ +âĺ ı +ïº ļ +ï» Ĥ +ðŁļ © +ìĪ Ł +Ë Ĭ +⤠µ +ðŁĴ § +ã ħį +ë© © +Æ ¬ +Î ĩ +âĩ § +âĵ ļ +ìĤ ¯ +ìĪ ¯ +ëĨ ĭ +âľ ¯ +ðŁļ Ģ +Ú ĺ +Ú ¨ +âľ Ń +ê² ħ +íĮ ° +íľ Ļ +ðŁĮ Ĭ +ðŁİ ĵ +ðŁĺ Ļ +Ë ĥ +ðŁĴ ģ +ðŁij İ +âĺ ¹ +ðŁĺ « +ðŁĴ » +ëĤ µ +ìĿ Ĭ +íĮ » +Ò ³ +á½ ² +âŀ ŀ +ëĤ ij +ëĿ Ī +ì£ ¤ +ï» ¯ +ðŁĩ © +ðŁ¥ ³ +âĴ ¼ +ðŁ¦ ĭ +âĺ Ĥ +ðŁĺ ° +ðŁĻ ĥ +ðŁĺ Ĵ +Û İ +Ï ķ +Ḡ¤ +ë£ ½ +ìĬ ¥ +ðĿij ī +É IJ +ðŁį İ +âķ ¯ +âķ ¹ +ຠ² +ï¾ ł +ë¹ ķ +ïº Ĩ +Ê º +Ó § +âĨ ł +ëĥ ĩ +ìİ Ī +ìŁ ¤ +ï± ¢ +âķ ¬ +âĺ ł +ðŁİ Ĭ +ãį į +ãİ İ +âĺ ° +âľ ĥ +ãħ ī +ë¯ Ī +ë¹ ¤ +ìı Ń +ðĿij ¢ +ðŁIJ ¾ +Å ĭ +ðŁij ¶ +âĶ Ľ +ï¿ ¢ +áĥ ¡ +Ä ¼ +Å Ĩ +Ñ IJ +ìĥ Ľ +ìĺ Į +ì± ¤ +íħ ģ +íļ ĥ +ï³ Ĭ +ðĿij Ķ +ðŁĩ « +âĭ ° +ðŁĺ ¨ +âĤ © +Õ ¬ +Ḡį +á» ´ +âĨ ĺ +âĺ ¯ +ãħ ı +ìł ¬ +âĻ Ķ +ðŁĶ Ķ +ðŁĺ ł +ðŁĻ Ĭ +à® ľ +á¹ ħ +âĹ IJ +âĿ Ī +âŀ ½ +ìĥ ħ +ðĿij ł +Æ ¢ +âĭ Ļ +ê° Ľ +ëĿ µ +ë£ Ł +ìı ľ +ïº ģ +ðŁĴ Ń +âĬ ĥ +ðŁIJ ° +ãħ Į +Ü ĵ +âŀ ķ +á½ ģ +ìķ ³ +ðĿij Ŀ +ðŁİ ¬ +É ¡ +à¤ Ĺ +áIJ ī +ì© ľ +ì¶ § +ï³ ī +ï» ħ +ðĿIJ ŀ +ठ¶ +ðŁĵ ¢ +ðŁį ĭ +ðŁĴ ħ +ï¾ ķ +⬠Ĩ +âĪ µ +ðŁ¤ ij +áĥ £ +Æ Ħ +Ñ ¹ +á¼ Ķ +ê° ł +ê´ Į +ê· IJ +ëĽ ´ +ì± ĺ +ï® Ń +ïº ¹ +ïº ¾ +âľ Ĺ +âĿ ¦ +ðŁij ¦ +áĥ Ĺ +Ù ² +á½ ´ +âĪ ı +âľ ® +ê¹ ° +ë² µ +ìĦ Ģ +ì© Ŀ +ïº ŀ +ïº ½ +ðŁĩ Ń +Ë Ĥ +ðŁį ij +ðŁį Į +ðŁĶ » +ê¹ ¬ +ìĬ Ń +ìľ · +ðŁĽ ij +Ç § +ë¼ Ľ +ïº ¡ +ïº º +ðĿij ļ +ðŁĵ ¦ +ðŁĶ İ +ðŁĹ ĵ +áĥ Ķ +âľ Ĵ +âľ ¡ +ðŁĮ µ +âĶ ķ +ëĢ Ŀ +ðŁį Ĭ +âĺ ĥ +ìĺ ħ +ঠ¬ +ðŁ¦ ģ +âİ ¯ +ðŁIJ ķ +Ñ ¿ +ॠ¤ +༠ĭ +ê· Ī +ì« Į +ðŁĩ ° +âĿ ī +ì« Ģ +íĿ Ħ +ðĿIJ ¢ +ðŁļ ¨ +âĻ ¤ +ðŁĺ © +ðŁį į +ðŁĺ ij +ðŁļ ļ +Ö Ħ +ë « +ë« ¼ +ठı +á¿ · +âĮ © +âĺ IJ +âŀ £ +ê¸ ± +ê¼ ¿ +ëĦ Ŀ +ìı ´ +ìļ ¤ +ì¿ ± +íİ IJ +ðŁĴ ¢ +ì´ IJ +âĩ ij +âĶ ĵ +âģ ¾ +Ü Ŀ +ðŁ į° +â´ ° +Æ ı +Ï Ł +Ú º +Û ĥ +áĦ Ĵ +âĪ Ł +âĿ į +ãĦ ² +ìľ ħ +ì¤ ı +ðŁĩ ² +êº Ħ +ðŁİ ¤ +âľ £ +⸠Ŀ +ï¸ µ +ຠ§ +áĢ Ļ +âķ ł +Õ ¯ +âı © +ðĿij £ +ðŁĴ £ +Å ĺ +ॠIJ +âģ ĥ +âĮ ĺ +ê» Į +ìĮ Ķ +ðĿij ĺ +ðŁ¤ ĵ +Õ ¿ +à¤ Ń +âĮ ļ +âľ Ŀ +ðŁIJ ¼ +Ë Į +âķ ļ +ï¦ Ĺ +âĿ ķ +âķ £ +ðŁIJ ± +à® ¤ +Ñ ¾ +ठļ +ठľ +ìĪ Ħ +ìļ ľ +ðŁİ ® +É Ĵ +Ú · +ຠį +âĨ µ +â Īĺ +âĿ Ĭ +ë¿ į +ìIJ Ī +ìļ ĺ +ì¯ § +íĥ ¯ +ìĸ ı +ï¸ ° +ðŁĩ ¯ +ðŁ§ ļ +ðŁĺ µ +ðŁĺ · +ðŁĮ ³ +ຠ¥ +Ä ī +Ä ¥ +âľ ¶ +á¿ ¾ +âĬ ± +âĺ ¾ +ê° ī +ê¼ ° +ëº ij +ðŁĶ Ĭ +ðŁĸ IJ +Å ¤ +Ò « +à® ® +âĮ Ī +âĹ Ĺ +ëĦ µ +ëħ ľ +ëľ ¹ +ðĿij ¥ +ðŁĴ ¿ +ðŁĽ Ĵ +Ê Ĵ +áŀ ĵ +ðŁIJ Ŀ +ðŁ¦ Ħ +ðŁį · +âĺ Ł +ï¸ ¶ +ðŁ¤ Ł +Ô ± +âĨ ² +âĪ İ +âľ « +ëĩ ½ +ëı IJ +ëķ Ħ +ï¦ ³ +ï§ Ŀ +ïº Ļ +ðŁij » +ðŁĵ º +êµ ¼ +ìĮ © +ðŁĮ ² +È ± +íĶ ķ +ðŁĺ ¤ +ãĮ ¢ +Ê Ķ +ठ¡ +á¼ Ī +ëİ ĥ +ë© ± +ë® Ī +ðĿIJ « +âĬ ķ +ëĥ ł +ë» ¬ +íĭ Ķ +Õ ¤ +á¼ ± +âľ ¥ +âĺ Ħ +âĪ ¥ +âļ ķ +ðŁij Ħ +ðŁİ ħ +ຠĻ +âĶ ¬ +á½ µ +Õ ¾ +Ö ģ +âĹ Ķ +ê¿ į +ëĸ µ +ë© İ +ë® ´ +ìķ ´ +áĥ ľ +á¼ ¡ +âĶ Ĭ +âķ ® +âĹ ¼ +ðŁį ¾ +ðŁĽ į +ðŁij Ĺ +ðŁ¤ ŀ +âľ Ħ +Õ Ģ +ঠ² +Ë ī +⣠¨ +Ä ¯ +Ï Ĭ +á´ ľ +ë¹ ³ +ï³ ĭ +ï¿ ł +Ä ª +âĤ ¸ +âľ ± +ê» IJ +ëĭ » +ë§ ¸ +ìŀ ¿ +ì© ¨ +ì ŃIJ +ì° ¿ +íħ Ł +ðĿIJ § +ðĿij ij +ðŁĮ İ +ðŁĵ ® +ðŁķ Ķ +âĹ Ļ +âĹ » +âŀ § +ìŁ Ŀ +âľ ¬ +ãĥ ° +âģ Ī +â ĵĺ +ðŁ ĴĮ +ï¬ ĥ +ຠĶ +ìĶ ° +ðŁĺ ª +× Ģ +ìĥ ¨ +ïŃ ĭ +ðŁį ķ +ðŁĺ ´ +Ï ³ +á¼ Ħ +á½ ħ +âĩ ¢ +âķ Ń +ìĺ » +íĬ ¤ +Ü ĺ +⤠´ +âĹ į +áŀ Ł +ðŁį º +áŀ ļ +ðŁı Ĭ +ðŁIJ · +Ê Į +á½ º +âģ » +ê½ Į +ëĪ Ĺ +ë Ĺı +ì¿ ° +íĢ ¼ +íį ħ +ï· ² +ðŁĮ ı +ðŁį « +ðŁį ³ +ðŁİ ° +ðŁij ° +ðŁĴ ² +ᥠĻ +ðŁIJ Ł +ï¿ ¡ +ðŁĹ £ +ðŁį ľ +âľ ² +ãİ ¢ +ðŁĶ ° +á¼ ¸ +á½ ij +Ä İ +áĦ Ģ +âĻ ķ +ëł Ŀ +ìĪ ´ +ïŃ Ń +Ó ľ +Ô Ģ +ëĢ ľ +ëĥ Ķ +ìĬ Ľ +ì« ij +ìº ¥ +ìº ¬ +ðĿij ¦ +ðŁĶ ¶ +ì¾ ¨ +ðĿIJ ļ +ðŁį » +ðŁĴ į +ðŁ¤ ¡ +ðŁķ Ĭ +â½ ĩ +âĵ IJ +ðŁį Ń +ðŁį ª +ðŁĶ Ĩ +Ò ¡ +á´ ĩ +É Ĺ +Ü Ķ +âĦ İ +âĿ ĥ +ëĹ Ģ +ï² Ķ +ïº Ī +ðĿIJ » +ðŁĴ Ĭ +ðŁļ « +Ñ ° +Ñ ³ +ठ· +âĹ ł +ðŁij ¤ +ï¾ ĩ +âĺ ĵ +ðŁį µ +ðŁ¤ ¨ +âĸ Ń +à® ´ +Ü ¢ +Ü ¬ +à´ ® +ðŁķ º +Ô ¹ +Õ £ +à´ ¯ +á ´Ģ +âĮ ī +âľ IJ +âŀ ¦ +ê¹ ½ +ëĮ ľ +ðŁı ¥ +ðŁĵ © +Ò ¹ +Ó ĺ +ठħ +âĿ § +Æ Ĺ +âĹ ½ +ðŁij « +ðŁİ § +ðŁij £ +âľ » +ðŁĻ ħ +ðŁĺ ĸ +ðŁĴ ® +ຠ° +ðŁĶ ľ +ðŁį Ħ +ðŁ¤ Ŀ +á ĥĿ +áŀ Ģ +âĩ ¦ +Ê ¾ +Ò ® +Õ ¼ +ठĨ +âĹ ħ +âļ ĵ +âļ ĸ +ê¿ © +ë¯ Ħ +ìIJ IJ +ìŀ ° +ì§ Ń +íĭ ĭ +íİ ¨ +íĻ § +ï² ij +ðŁİ Ĺ +Ù ³ +ðŁij ¸ +ঠ® +ðŁij ķ +Ú µ +âĢ ¾ +âŀ ° +ðŁij ¯ +ðŁİ ¼ +ðŁı ģ +Ä º +Ê ı +Ú ³ +âı ± +ê½ Ī +ëĿ Į +ìĮ ī +ìĹ · +ìŀ ´ +íĹ ¹ +íľ ¨ +ðĿĹ ² +ðŁĮ IJ +ðŁİ Ļ +ðŁı µ +íĽ Ļ +ðĿij ħ +ðŁĺ ¶ +âĵ ħ +âķ ¥ +ðŁį ı +ï¦ İ +Õ © +ðĿIJ Ħ +Ó £ +Ú ¿ +âĻ ļ +ðŁĶ Ĺ +Ḡ« +âĭ ® +âĸ ¦ +⼠½ +âľ µ +ãħ Ĩ +ãħ Ĭ +ëĦ Ļ +ëĿ ¨ +ë¥ Ħ +ìĦ ¦ +ì§ ° +ì§ ¹ +íī Ī +ï§ ij +ï» ĩ +ðŁĮ ¾ +ðŁı ĸ +ðŁIJ ij +ðŁĴ ³ +ðŁĵ Ĩ +Û ĩ +Ü ķ +á½ ½ +ëĦ ľ +à´ ² +à´ ³ +àº Ń +áĥ Ľ +âĿ Ķ +âij ħ +áĥ ¥ +ðŁĵ ħ +âŀ ³ +á´ µ +ï¹ ¡ +ï¹ ¶ +Î Ĩ +ठ¥ +áī µ +âĿ Ļ +âĿ ± +ëī ł +ëİ ł +ëı Ľ +ë¿ ħ +ìĶ ¸ +íij ¯ +íŀ ī +íŀ Ľ +ï§ Ħ +ïŃ ĺ +ïº ¦ +ï» ¸ +ðĿij Ĥ +ðĿij ı +Ï ij +Ú ł +áĢ Ķ +áŀ Ķ +á¹ ¢ +ëĦ ¸ +ðĿIJ ¨ +ðŁĩ ´ +Õ ° +ðŁij ł +ðŁį Ĩ +ðŁı Ģ +ðŁ ijIJ +ðŁį ĩ +ðŁIJ £ +áĪ Ń +Ü ª +ðŁ ĮĢ +áŀ ĺ +âĩ Ħ +ðĿIJ Ģ +Ê Ļ +âĶ ¼ +ðŁı ¿ +Æ · +È ł +Ñ ½ +âĤ ¨ +ê´ Ń +ê¹ » +ëĶ ¨ +ìĪ Ģ +ì¾ ° +íĨ Ī +ï® § +ï¯ ½ +ðŁĶ ħ +ðŁĶ ® +Å ¢ +Ê ° +Ñ ¸ +ठ£ +âĬ Ĺ +ëª Ħ +ï¹ · +ïº ħ +ðĿIJ µ +ðŁĮ ¶ +ðŁĵ ° +ðŁĶ · +ðŁĸ Ĵ +ðŁ¤ ² +ëī © +ðŁİ Ĩ +ðŁ§ IJ +ðŁį ® +âĨ º +âĿ ¢ +ðŁij ª +ðŁij ± +âĨ ¡ +áŀ ı +Ú ķ +ðŁį ¹ +ðŁĴ Ģ +Ë ® +Ó ¨ +Ö ħ +ठĩ +âĤ ¡ +âĪ ķ +âĺ ī +ê¹ ¼ +ê¼ IJ +ì½ ¸ +ðĿIJ ¬ +ðŁı ħ +ðŁij Ļ +ðŁĴ ī +ðŁ¤ Ļ +È ĺ +É ³ +É ¹ +Ù º +áĢ Ħ +á¿ ³ +âļ ĺ +âĿ Ĩ +ëĨ ī +ìĸ į +ìĺ ĩ +ì¥ ĺ +íĸ ħ +íĻ ij +ï® Ĭ +ï¿ Ń +ðĿĴ IJ +ðĿĹ ¢ +ðŁĶ ĸ +ðŁĶ ¨ +ðŁļ ij +ðŁļ ² +Æ ¸ +âĹ ¥ +ðĿIJ Ń +ðŁį ½ +âĹ ij +âĵ ĩ +ðŁĶ ± +âľ ¼ +ï¹ ĥ +âķ ± +ãĢ Ĺ +ðŁı ĭ +ðŁļ ´ +ðĿIJ ® +Ä ļ +Õ ı +Ä ¶ +áĥ ij +á¹ ¬ +Ä Ī +Ä Ĵ +Ò ° +Ó ķ +â IJ +âIJ £ +âĹ ¢ +âļ Ļ +ãħ Ĺ +ê° ¬ +ê³ ª +ê» Ģ +ëĦ ´ +ëİ ģ +ëĿ Ķ +ë¬ ½ +ëŃ į +ìĩ ³ +ì° ¹ +íĮ ¹ +íŀ Ŀ +ï® ĭ +ï ¶Ī +ðĿĴ Ĥ +ðŁ¥ Ģ +ðŁ¦ ħ +Ê ĺ +á¼ ij +âģ İ +ðŁį ŀ +âĨ ĸ +âĨ Ļ +ðŁİ ĥ +âĦ ¡ +âĭ ± +ðŁĶ į +ಠ¨ +áµ ĥ +âĶ « +⦠¿ +ðŁĩ » +Æ ¤ +Ò ı +Ò · +Û ī +à® ķ +Ḡ³ +ï¬ ± +ðŁĨ Ķ +Ú Ń +Û ¦ +áħ ¡ +âĦ ¹ +ê¿ İ +ëķ Ķ +ë¼ ī +ìļ § +ì² µ +ì´ ¨ +íĬ Ī +íĸ IJ +ðĿĹ ĺ +ðŁĩ ¿ +ðŁİ ĸ +ðŁij ħ +ðŁ ĵĺ +ðŁļ Ļ +ðŁĽ µ +ච½ +⼠µ +ðĿIJ ³ +ðĿIJ ¸ +âļ Ķ +ðŁij Ń +Ó ij +âĶ ¯ +ðŁħ ¿ +ðŁĺ ¹ +ï¿ « +â¼ ¤ +ðŁĴ ĩ +ðŁĵ İ +ðŁĸ ĭ +ঠ¸ +ðĿIJ į +Ä ² +Ï ĭ +Ñ ¬ +Ú ¬ +Ü Ĵ +á´ ¬ +ï¨ Ħ +É £ +Ë ij +Ï µ +Ò Ŀ +Û ¥ +Ü ł +๠Ľ +áĥ ķ +áĬ ķ +á¾ ¶ +âĤ · +âĩ ¾ +âķ © +âĸ IJ +âĺ ª +âĺ ® +âĿ ļ +âĿ Ń +âŀ ± +âµ İ +ãı Ĭ +ë© ĵ +ìĹ ¾ +ìª Ħ +íĵ Į +íķ ¼ +ïŃ ¬ +ðĿij Ĩ +ðĿij ŀ +ðĿĸ Ĭ +ðŁİ ¸ +ðŁı Ħ +ðŁij µ +ðŁĴ ł +ðŁĶ ĺ +ðŁ¥ Ĥ +Å ª +à· ĥ +á´ ¼ +âĬ ° +ë³ ı +ë´ £ +ï¥ ľ +ðŁĵ Ī +ðŁķ ¯ +ðŁ§ Ģ +âĻ IJ +ðŁĨ Ĺ +ðŁĵ ķ +ðŁ§ ģ +Ü « +âĿ IJ +Õ ķ +འķ +âŀ Ŀ +ঠķ +ðĿIJ ¶ +É ¢ +Î Ħ +áĨ ¢ +âĤ ± +Õ į +à¡ ķ +á´ ° +Ḡ© +⼠· +âĿ ® +ê¡ ĵ +ëı ¤ +ëĹ IJ +ëµ Į +ìij Ī +íı ¿ +íĹ µ +ðĿIJ İ +ðŁĨ ĺ +ðŁı Ł +É ¥ +Õ » +à¡ Ķ +ठĸ +á´ ¸ +âİ Ļ +âİ ¥ +âı ³ +ëģ ķ +ëĬ ī +ì¡ į +ì¹ ¡ +ï¦ ¶ +ï¬ Ł +ï® « +ï® ¯ +ï± ĥ +ï ·» +ïº µ +ðĿĹ Ķ +ðĿĹ ¡ +ðŁİ ¨ +ðŁĶ Ĵ +Ú Ľ +ठ§ +âŀ ¹ +áĢ Ģ +ðŁį ħ +âĹ ¤ +ठł +ðŁIJ ¥ +áĥ Ĵ +ðŁı Ŀ +ðŁį ¼ +ãĮ § +âĿ Ľ +ðŁIJ Ī +ঠ¯ +áĢ ŀ +ãĢ ĸ +áŀ Ļ +ঠª +Õ Ĩ +âĬ Ĩ +âľ ¾ +ðŁIJ Ĺ +ï¹ ¿ +Ä ¦ +Ü Ł +ಠł +ಠ¥ +áŀ ī +á´ ¥ +á´ © +á½ Ģ +á½ ¡ +âĨ ķ +âŀ ¯ +ê¡ ij +ëij £ +ë± Į +ìĪ ij +ìľ Ķ +ìŀ ½ +ì¨ į +ðĿij Ģ +ðŁĮ Į +ðŁį ¦ +ðŁį © +ðŁIJ ļ +ðŁĵ Ĵ +ðŁĵ ¹ +ðŁ¥ ij +Ä ĭ +Ë Ĺ +Ñ « +Õ ¢ +Ú ° +â ĮĢ +âĹ Ĥ +âĹ £ +âľ Ľ +âĿ Ĵ +âĿ ĺ +âŀ Ļ +âŀ ² +ãİ į +ê¡ IJ +ëŀ ĸ +ìĬ Ŀ +ìĽ ¤ +ì¡ ĭ +ì¨ ° +íĹ Ļ +ï¥ ¸ +ï³ į +ï» İ +ðĿij ĵ +ðŁĵ Ĭ +ðŁļ ¼ +ï¦ ģ +ðĿķ Ĵ +ðŁ ijľ +ðŁij ¿ +ðŁĩ ½ +à· Ħ +âĸ ´ +ãį ī +âĬ ĩ +ðŁ§ ¸ +Ú ¡ +â¾ ĥ +ðŁĹ » +âĵ ij +ðŁ¤ ¸ +ðŁ¤ ¯ +êĴ ° +ðĿIJ ĵ +âĶ ´ +êĴ ± +áĢ ĺ +â ĽĦ +ï¹ ¹ +Ó Ķ +áĥ ± +Ü ¡ +ß ŀ +âĻ ı +âľ ¸ +ìij ¨ +ðĿIJ Ŀ +ðĿIJ ¥ +ðŁį ī +ðŁij ¼ +ðŁ¥ Ŀ +Æ Ķ +Ý ¬ +ठ« +ຠļ +á´ ´ +á½ ĸ +âĤ ¶ +âİ ¢ +âĿ ħ +⣠« +ãİ Ľ +ë® ¨ +ëº Į +ë¼ ĺ +ìĨ Ŀ +ìľ ³ +ìŀ Į +ì£ Ĺ +ìª ĺ +ì» ¹ +ï· ¼ +ïº Ĥ +ðĿIJ ´ +ðĿIJ ¼ +ðŁĮ ļ +ðŁı « +ðŁĴ ¤ +ðŁĴ ¶ +ðŁĴ ¼ +Ê ķ +Ê ½ +â² Ł +ãī ł +ê¡ Ĵ +ëľ Ģ +ìĥ ¾ +ì¸ ¤ +ï¥ ģ +ðĿļ Ĭ +ðŁļ ĥ +âŀ Ľ +ìħ ´ +áĦ ĭ +âĩ Ĺ +ï§ · +âĺ ĸ +ðŁIJ ¦ +⸠ľ +ðŁĴ ´ +ðŁ¤ ļ +ãĬ Ĺ +âĮ Ľ +áĪ Ľ +༠º +â½ ī +ðŁı ¢ +âĵ ŀ +âĺ ½ +ãĢ Ļ +ðŁ¤ ® +Å IJ +áĥ ¬ +ðĿĹ » +ðŁį ĸ +Æ Ĭ +Ê Ł +ß ĭ +ठĭ +áµ Ķ +á¿ ĥ +âĦ ī +âĮ ĭ +âı ² +âĵ Ī +âĵ ¢ +âķ Ķ +âļ ij +âĿ ĭ +âĿ İ +â µľ +âµ £ +ëĴ Ī +ëľ ģ +ë¶ ĩ +ìį » +ìĺ Ń +ì§ ¢ +íĹ Ģ +ï§ Ĭ +ï ¬¸ +ï± ¡ +ðĿIJ º +ðĿij § +ðĿĺ ¦ +ðŁĵ ¥ +ðŁĺ Ł +ðŁ¥ IJ +Ä ĸ +É ¨ +áĢ IJ +áĥ ĵ +Ạĵ +á¼ ¶ +á½ Ħ +âĤ ¤ +âĮ ľ +âĮ Ł +âİ ł +⼠¸ +âµ į +âµ ı +âµ ĵ +ãĢ ĺ +ë ·¸ +íħ ¼ +ï¦ Į +ïŃ Ħ +ïŃ İ +ðĿĻ ļ +ðĿļ ĺ +༠ĵ +ëŃ ħ +áIJ Ľ +ãİ ¾ +ï¨ Ģ +ðŁĹ ½ +âĻ ŀ +Ë ĸ +âĹ ŀ +ðŁ¤ « +ðŁĺ Ĺ +ï½ ¦ +ðŁ¤ ¢ +âģ ĩ +ãĢ µ +ðŁį Ķ +áĬ ł +ðŁĺ ¼ +ðĿĹ ® +ðŁIJ ³ +ðĿIJ ĭ +ðŁĨ ļ +ðŁĶ Ľ +Ñ » +Ü ¨ +à® ² +âľ ŀ +âµ Ļ +êµ £ +ì¸ ¨ +ðĿ IJľ +ðĿĺ ° +ðŁĶ ½ +Ç » +Ç ¿ +Ê ĩ +Î IJ +Ð Ģ +Ñ ¡ +Ñ ² +Ò Ĵ +Ù ¶ +ß ķ +ච± +áIJ ģ +âģ ŀ +âĸ § +⼠Ī +âľ ľ +âľ ¹ +⣠¹ +⤠ĩ +ê² Ĭ +ê¾ ľ +ë¯ IJ +ë³ IJ +ìħ © +ìIJ ¬ +ìij ¹ +ï¤ Ķ +ï¦ ļ +ï¬ ł +ïŃ Ķ +ïº ¶ +ðĿĴ ı +ðĿĸ Ĩ +ðĿĹ ¶ +ðŁı Ĥ +ðŁIJ ½ +ðŁĴ © +ðŁĵ ½ +ðŁĹ ¨ +ðŁĹ º +ðŁĺ ¸ +ðŁ¥ § +Å Ĺ +Ê İ +Ò Ļ +× ² +ठĪ +á¼ ´ +á¿ ij +âµ ī +ãħ ĵ +ì½ ´ +ðĿĸ ĵ +ðŁĵ Ĺ +ðŁĶ ª +ðŁĸ į +Ï Ĵ +ðŁij ¬ +áĥ Ļ +âĨ ¬ +âĶ ¤ +⼠¹ +âĻ Ł +ðŁļ ¶ +ðŁij ¾ +âĪ ĭ +ðŁIJ ¯ +à¼ İ +âľ · +ï¨ Ļ +âĶ » +ðŁij ¹ +áĦ ī +ຠª +â¾ ı +â½ ħ +ãİ ĸ +Ñ ´ +Õ ® +Ú ¼ +áĢ ķ +áĨ ¼ +ëŃ ı +ðŁIJ ¸ +ðŁļ £ +Æ Ŀ +Ô » +áĥ ¢ +ðŁį ¯ +É ¦ +Õ ¦ +âĻ ĭ +ï¬ « +ðĿĹ ¦ +Ç ļ +É ± +ठī +á´ Ħ +âĻ ĵ +⼠° +⣠ª +ëĥ ĺ +ë¢ ¸ +ìĤ ij +ï® Ķ +ðĿķ ĸ +ðĿĹ § +ðŁĩ ¼ +ðŁĵ ĭ +ðŁļ ľ +ðŁ¥ ¤ +Ä ® +Å · +ß Ĭ +ॠ¥ +à® ª +áŀ Ħ +áµ Ģ +Ḡħ +á¼ ¢ +âĪ Ŀ +âĬ ¹ +âĴ ¶ +âķ ´ +⼠± +⼠³ +⼠º +âŀ Ł +ãı Ħ +ê¸ Ķ +ê¹ Ł +ëĩ ° +ë¹ » +ìĤ ¥ +ìĽ » +ì° Ł +íĥ ° +íĨ º +íļ ½ +ï¤ ´ +ï¥ ¾ +ï³ Ŀ +ðĿIJ ¦ +ðĿĴ ľ +ðĿĴ Ł +ðĿļ Ĺ +ðŁİ Ń +ðŁı ĵ +ðŁı ³ +ðŁı º +ðŁIJ į +ðŁij ĥ +ðŁĴ ı +ðŁ¤ ĸ +ðŁ¤ µ +Õ ² +âµ Ķ +ëĺ ¬ +ï¦ £ +Ê Ĥ +áĨ « +áŀ ij +ðĿĸ İ +ðĿĹ ĸ +áĦ ĥ +âĩ ł +áĢ ¡ +འĦ +âŀ ¸ +ï¦ Ļ +âĩ ļ +ðŁIJ ¬ +ðŁIJ ¢ +â¾ Ĵ +ðŁIJ ¤ +ðŁĶ « +ãĢ ŀ +ï¸ º +ðŁĺ º +â½ ´ +ðŁĨ ķ +âģ ¿ +ðŁį ¨ +ಠķ +ðŁļ ĺ +áŀ ħ +ঠħ +áŀ ¢ +ਠľ +â ļĮ +ãĢ ½ +à· ´ +âĵ Ľ +áĢ ľ +ìĨ ¨ +Ë © +Ü Ĺ +âĭ ¼ +ðŁĻ ī +Å Ĭ +É ĵ +Ê ² +Î ° +Ñ ¼ +Ô ¿ +à¡ IJ +༠ľ +འ¦ +ᶠľ +âĤ ² +âĨ ¨ +âĬ ¥ +âķ § +âĻ ľ +ãĭ ¡ +ë´ ¬ +ë¶ ij +ìī ¿ +ìİ ħ +ìł ± +ì° § +ï² ¡ +ðĿĴ Ľ +ðĿķ £ +ðĿĹ ľ +ðŁį ² +ðŁİ © +ðŁIJ IJ +ðŁIJ ł +ðŁij ½ +ðŁĴ ij +ðŁĵ ľ +ðŁķ µ +ðŁ ļĮ +ðŁĽ £ +Ê ĭ +Ó ¯ +Ù ¸ +ß Ķ +ß Ļ +à¡ ĵ +á´ į +Ḡ¿ +âı º +âĸ ¥ +ë¤ ½ +íľ ij +ðĿIJ ¹ +ðĿĸ Ķ +ðĿļ İ +ðŁĵ Ħ +ðŁ¦ · +Æ ĥ +ঠŁ +âĮ Ĥ +âĺ Ń +â² ļ +ëĿ ķ +ðŁİ £ +à® ĩ +འĨ +áħ µ +áĹ ľ +âĢ ½ +âĮ £ +âģ ½ +ðŁĵ ¬ +ðŁ¤ § +âĩ ª +â½ £ +âĹ Ł +ï¨ Ĺ +êĴ ª +ðŁĽ Ģ +Ç Ĥ +ðŁ¥ ¶ +ðŁİ į +ï¿ © +ðŁij Ĵ +áµ Ī +ï¸ ¿ +áħ © +â¾ ¦ +à° ¤ +á´ ĸ +ਠ¬ +àº Ĺ +༠» +Ñ º +ਠª +á´ ³ +ðĿIJ Ī +à» Ģ +á´ ¿ +âĤ į +âĩ ¡ +⼠ª +ðĿIJ Ĥ +ðĿĴ ķ +ðŁ IJľ +Ê į +Ñ ± +འĥ +ë® IJ +ìĽ ¡ +ìľ ģ +ðĿIJ ¿ +ðĿķ ł +ðŁij Ľ +Æ ª +Ï º +Ó ¬ +Ù ¿ +Ý £ +ઠī +à® ¹ +འij +áĨ ¯ +áµ ĩ +âĩ ¥ +âı ª +âĻ ° +âļ Ń +âļ ¾ +ãħ Ħ +êĢ ° +ê° Ĺ +ê² ĭ +ê² » +ê¶ ľ +ê¼ ĩ +ê½ ¹ +ëĤ Ł +ëħ Ī +ëĭ ¢ +ë§ Ł +ëª Ĩ +ëµ Ģ +ì½ ± +íĩ ĺ +íľ ľ +ï§ ¾ +ï± µ +ï² ¢ +ï² ¤ +ðĿĴ Ĭ +ðĿĺ ¯ +ðŁį Ĺ +ðŁı į +ðŁIJ ĺ +ðŁĵ ¡ +ðŁĶ ŀ +ðŁ¤ ³ +ðŁ¥ ģ +ðŁ¥ Ĺ +ðŁ¦ Ĭ +Ä µ +Æ ¦ +Ç µ +É ¯ +Î ı +Õ Ħ +Ü ¥ +འģ +ᨠł +âķ « +ãİ ī +ë· ´ +ìĨ İ +ìİ Į +ì£ µ +íĽ ł +ï§ ª +ï³ ı +ï» º +ðĿij ģ +ðĿij ĩ +ðĿĴ Ĩ +ðŁİ ł +ðŁIJ Ķ +ðŁij Ł +Å ĸ +ठĮ +á¾ ½ +ê¦ Ĵ +à® Ł +á´ ± +ðŁı ° +ðŁIJ ŀ +འĢ +áĢ ħ +âĬ ¿ +ðŁIJ § +ἠģ +â¼ Ī +âĶ ¿ +ðŁ¥ ´ +â¼ ¿ +ðŁ§ ľ +ãħ ¿ +âĦ « +ãĢ ³ +ãĬ Ļ +â¼ Ģ +ï ¦¬ +ðŁı ¬ +ðŁĵ » +áĬ Ľ +áĦ ħ +ຠĬ +ຠĽ +áħ ³ +ðŁij ® +à® ± +âĺ ĩ +ðĿIJ ı +à´ µ +à» ģ +འı +འ¢ +ᥠ± +âĤ £ +ï¥ ¦ +ïŃ Ļ +ï´ © +ï¹ Ĥ +ðŁį £ +ðŁķ ¹ +Ï ĸ +ච¸ +ຠ¢ +áĭ Ń +âİ Ŀ +âĹ Ŀ +âĻ Ī +âĻ İ +ê½ ¥ +ì³ Ķ +ì¼ ij +ï± ° +ðĿij ĥ +ðŁĮ ª +ðŁį ¡ +Å İ +Ê ¦ +Ñ § +Ó İ +Ô ´ +Ú Ī +ß ĵ +ß § +ठĶ +áĪ « +áĪ µ +áĹ © +á´ ł +á¼ ł +âĢ Ĺ +âģ ij +âĦ ı +âĸ ĩ +â² £ +ãĦ ³ +ãī ® +ê³ Ĺ +ëĦ Ĵ +ëĸ « +ë¡ Ħ +ë¹ ° +ë½ ģ +ìĦ ģ +ìĮ ĺ +ìŁ Į +ì³ ī +ì¼ ķ +ï¬ » +ï³ İ +ï¹ ¸ +ï¹ ¾ +ðĿIJ Ĩ +ðĿij · +ðĿĽ ¼ +ðŁİ ı +ðŁİ ŀ +ðŁIJ Ļ +ðŁij Ĥ +ðŁĵ ģ +ðŁĸ ± +ðŁļ į +ðŁļ § +ðŁĽ ¡ +ðŁ¤ Ĵ +ðŁ¥ ŀ +ðŁ¥ © +ðŁ¦ Ģ +ðŁ¦ ĸ +Ë ¢ +Ü ļ +à® µ +áĢ ģ +áī ° +âı Ń +âĻ ¿ +ê³ ĺ +ëı Ŀ +ëķ ĥ +ìħ Į +ìĴ ¸ +ìĽ Ł +íħ Ħ +íľ « +ï§ ĺ +ï¿ ¬ +ðŁı · +ðŁĶ § +ðŁ¥ Ī +Æ ĸ +áŀ ĩ +áŀ ĸ +âģ º +âĹ ľ +âŀ © +ê¦ Ń +ëĻ ¤ +ïŃ ¼ +ðĿĻ ĸ +ðĿĻ £ +ðĿĻ ¤ +ðŁĮ Ŀ +ðŁĶ ij +ðŁĽ ł +ຠĩ +âĺ £ +ãĦ ¨ +ðĿĸ Ĺ +Ó ĵ +âĨ £ +ðŁ¥ ī +ðŁĮ ł +ðŁĺ ½ +ãİ ł +Å § +ðŁIJ Ĵ +ï§ IJ +ðŁĺ ¿ +âĪ ¬ +ðŁIJ ® +⣠± +ಠ¡ +â¾ ¼ +à° ² +Ë ¶ +âĸ ¿ +Õ Ī +áŀ İ +áħ ¥ +áŀ Ĺ +Õ § +ðŁ¤ IJ +ðŁį ł +ঠ¤ +චº +âĻ į +ìĺ Ļ +íĺ ĵ +ï¹ º +ðŁĽ ³ +Å ī +á´ İ +âı ľ +âĶ ³ +ê¸ · +ì¡ Ķ +ðĿĴ Ī +ðĿĴ į +ðĿĴ ¹ +ðĿĵ ĩ +ðĿķ Ł +ðĿĹ ¹ +ðŁĮ ħ +ðŁı ´ +Ä Ķ +Ä ¤ +Å µ +Ç ¾ +Ï ŀ +Ï ¶ +Ô ³ +Ü Ĩ +ß © +à¡ Ĵ +ठĺ +චļ +འĸ +áģ Ĭ +áĥ ŀ +áĦ Ĥ +áĭ « +á´ º +Ḡ£ +Ḡª +á¹ Ĥ +á¼ · +á¿ ĩ +âĩ Į +âı ¬ +âĻ Į +â® Ł +â´ » +âµ Ł +ê¦ ķ +ê¦ ª +ê¦ ® +ê² Ħ +ê¾ IJ +ëĥ ij +ëķ ĭ +ë¡ ¸ +ë¬ Ģ +ìĩ ¤ +ìĪ © +ìľ ķ +ìŃ ĺ +ì· ° +ì ·¸ +íľ Ģ +ï¤ £ +ï§ į +ï± Ħ +ï³ ij +ðĿIJ ¤ +ðĿĴ ĵ +ðĿĴ ¶ +ðĿĹ ¼ +ðĿĻ Ĭ +ðŁĩ ¾ +ðŁĮ Ľ +ðŁĮ ® +ðŁİ ĩ +ðŁİ ² +ðŁı Ľ +ðŁij ¥ +ðŁij ´ +ðŁĴ Ĩ +ðŁĵ Ĥ +ðŁĵ § +ðŁķ IJ +ðŁĸ ķ +ðŁĺ § +ðŁĻ Ģ +ðŁļ Ĵ +ðŁĽ « +ðŁ¤ ł +ðŁ¥ ļ +ðŁ¥ Ľ +ðŁ¥ £ +Ç ¯ +È § +Î Ĭ +Ò ² +× ° +Û ij +áĥ © +áĦ Į +áĪ į +áī ¥ +áı Ĥ +âģ ± +âĬ ¢ +âĹ ĵ +âĿ ° +ë¿ ¡ +ìĽ © +íģ Ń +íĨ ³ +íĬ Ħ +íĵ ¸ +ï¥ £ +ï¥ ´ +ï± IJ +ï± ¯ +ï³ ļ +ðĿĸ ĺ +ðĿĺ Ģ +ðŁIJ Ĭ +ðŁIJ Į +ðŁij ļ +ðŁĵ ĥ +ðŁļ Ľ +ðŁļ ª +ðŁ¤ ° +Ä ´ +áĥ ® +áĹ ¨ +âĻ ® +â² ŀ +ãĪ Ķ +ì ħį +ãħ ĥ +ï¥ ¡ +ຠ¡ +Õ İ +Õ º +⬠Ľ +â½ ¤ +ðĿIJ ² +âŀ µ +áĢ Ľ +âĶ ħ +âĨ Ł +â¼ Ĭ +ðŁĮ ½ +ðŁļ ¿ +ï¦ Ĭ +ãĦ £ +⼠© +ï© Ľ +ðŁį ± +â¾ ¨ +à´ ¤ +áŀ ģ +ຠŀ +Ê ļ +ðĿIJ Ĵ +à´ ± +áŀ ľ +à® © +à° Ĺ +à´ ļ +âĩ £ +ï¦ ķ +Õ ħ +Æ ĺ +âĤ ¦ +âĶ Ħ +ï¦ Ł +ï¦ « +ðĿIJ ģ +ðĿIJ ĥ +ðŁį ¸ +ðŁIJ ² +Å ¶ +É ĸ +ß ĺ +ภ¦ +འĶ +áĨ · +âģ ķ +âĵ Ĥ +âĿ ľ +ï¥ ¥ +ï¬ ® +ðĿĹ Ŀ +ðĿĹ ¿ +ðŁİ ¾ +ðŁĹ Ŀ +ðŁ¦ Į +Æ ħ +Ç ª +Ò Ĺ +Ü Ľ +ß ł +à¡ ij +áī £ +áĬ Ń +á¹ ¡ +âŀ ¼ +âŀ ¾ +â´ ± +ãī ¡ +ê³ ¯ +ë½ Ī +ìĤ ĺ +ìī ij +ì «ĺ +íĮ ĥ +íĻ ° +ï¤ Ĺ +ðŁĮ ¬ +ðŁĮ ° +ðŁį ¤ +Ä » +Å ĩ +Æ ¨ +É ķ +Ò ¢ +Ò º +Ö į +× ± +Ú ± +Ú ½ +Û IJ +ठĽ +à· Ģ +๠ļ +ຠ« +á´ ¹ +á ½Ķ +á¾ ³ +âĤ Ĵ +âĨ ´ +âĩ Ŀ +âī ħ +â Į¨ +âĵ ĵ +âĸ ¢ +âļ ¬ +âŀ Ń +â² Ĵ +ãİ ¿ +ê¿ ´ +ëĪ ± +ëį ¬ +ëİ IJ +ëIJ « +ëĶ « +ë± ģ +ìĥ ¥ +íĮ ¼ +ïŃ ĵ +ï® ¥ +ï² ° +ðĿIJ ĩ +ðĿIJ ij +ðĿij Į +ðĿĵ ª +ðĿķ ļ +ðĿĺ ª +ðĿĺ ¼ +ðĿļ Ľ +ðŁĩ ¶ +ðŁĮ Ħ +ðŁĮ ķ +ðŁĮ ¤ +ðŁĮ § +ðŁį ¬ +ðŁİ ĭ +ðŁİ » +ðŁı ¨ +ðŁIJ ĩ +ðŁij ĵ +ðŁĵ IJ +ðŁĵ Ļ +ðŁĶ ¼ +ðŁķ Ĵ +ðŁĸ ı +ðŁĸ ¥ +ðŁ¤ ¬ +ðŁ¥ Ĭ +ðŁ¥ Ĵ +ß Į +ຠĦ +á¼ µ +âķ ¡ +â² ¤ +â´ ¼ +âµ ¢ +ãĪ ¯ +ëĵ ¸ +ëŁ ĩ +ëº į +ðĿĻ § +ðŁį Ī +ðŁĶ ¬ +ðŁĸ Ĭ +ðŁ¤ ¾ +Ë ¡ +Ü © +âĮ ¡ +âŃ ij +â² ¦ +ë© ī +ì¼ Ń +ï¿ ¤ +ðĿĴ İ +ðĿĹ ¥ +ðŁIJ µ +ðŁķ ¶ +ðŁķ ¸ +ðŁ¤ ľ +Õ ª +áĪ ĭ +ðŁ¥ µ +ï° ģ +áµ IJ +âķ ĵ +áĢ ĸ +âĭ Ī +É ŀ +âŀ ® +ॠ° +ãĨ ģ +ðŁĴ ± +ðŁı Ń +áĨ ¨ +ðŁį ļ +ðŁ¦ IJ +á´ » +âĺ Į +à´ ķ +Õ ± +áħ ® +ðĿIJ Į +Å ¦ +ຠķ +âľ Ļ +Ë ³ +Ô µ +âķ Ĵ +ðĿĹ Ĺ +ðĿĹ ł +Ú ļ +ঠ§ +âĨ Ŀ +âĻ ī +ãĮ » +ì¹ Ĭ +ðĿĹ º +ðŁ§ ĺ +ì³ £ +ï¬ Ŀ +ðŁij º +Ç Ł +Î Ī +Î « +Ñ ¥ +Ô ² +Õ ¨ +Ü ¦ +ঠĨ +ঠ¥ +áIJ ¢ +á¼ ģ +á¼ ĺ +á¼ ¦ +âĵ Ŀ +ãĪ ° +ãİ Ĺ +ê² ¡ +ë¨ Ģ +ì£ Ķ +ì´ ¤ +ìµ Ŀ +ï§ ´ +ïŃ Ĭ +ï² Ł +ðĿIJ · +ðĿij ĭ +ðĿĵ ī +ðĿĺ µ +ðŁĴ · +ðŁĽ © +ðŁ§ ¹ +Å Ķ +Ê ŀ +Ë ¥ +Î Į +Ñ © +Ó IJ +Ó ł +Ú ij +Ú Ĵ +ß ¨ +ઠĪ +áIJ ĥ +á¹ ¯ +âĤ ĭ +âĤ µ +âĦ ħ +âĦ ł +âĪ £ +âī º +âī » +âĬ Ľ +âĮ IJ +âİ ĵ +âĺ ¸ +âĻ Ĵ +âļ Ĵ +âľ ĩ +âľ ł +â´ · +âµ ĸ +ãĦ ¸ +ãī ¢ +ãī ° +êĩ ´ +ê´ ¸ +êº ł +ëĤ ı +ëĤ ¢ +ëIJ Ģ +ëº ´ +ìĥ ľ +ìį ħ +ì¤ « +ì± ¦ +ìº ij +ì¼ ģ +ì¿ ³ +íĤ ģ +íħ ¡ +íĴ Ĥ +íĴ ī +íľ Ħ +ïŃ ª +ï® ¬ +ï¯ ¦ +ï± ª +ï² ı +ï ´Ģ +ï» Ĩ +ï¿ ¦ +ðĿij Ĺ +ðĿĸ Ļ +ðŁĮ ¡ +ðŁį Ŀ +ðŁį § +ðŁİ « +ðŁı ĺ +ðŁı ª +ðŁIJ ĭ +ðŁIJ Ľ +ðŁIJ º +ðŁij ĸ +ðŁij ŀ +ðŁij · +ðŁĵ Ģ +ðŁ ĶĦ +ðŁĶ Į +ðŁķ Ļ +ðŁĻ į +ðŁĻ İ +ðŁ¦ į +Ç ° +É Ł +Ê Ĩ +Ô ¼ +Ú ľ +ঠ¡ +ঠ¶ +áĴ ĥ +á¼ © +âĵ ķ +â² Ī +ê° ° +ê¹ ł +êº ħ +ëĦ ¹ +ë¯ ĵ +íIJ Ī +ï§ ¶ +ï® ij +ï² ¨ +ðĿĴ ī +ðĿĴ Ķ +ðĿĹ ¨ +ðĿĻ ŀ +ðĿļ Ĵ +ðĿļ ķ +ðŁIJ İ +ðŁ¤ ķ +ðŁ§ Ķ +Ï ° +Ô Ŀ +âĮ Ĭ +âĴ ¾ +ãī £ +ïŃ © +ðĿļ ŀ +Ê ij +ঠ¦ +áĦ ĩ +âī ĥ +â² Ģ +ìŁ İ +ðĿij ¶ +ðĿĵ ² +ðŁ İ· +ðŁļ ¹ +ຠģ +áł ł +ãĦ ļ +ðŁIJ ¿ +ἠļ +âķ ³ +ðŁIJ Ń +âĴ ¹ +ðĿĸ ļ +âĻ ĸ +ãĪ ² +âĨ ¾ +áĦ Ĩ +âķ Ľ +ðŁ¤ į +â½ ¥ +ðŁ Į¨ +âĪ ® +ãĮ ĺ +ãį ij +ï¹ Ģ +âĵ Ĺ +âĬ Ħ +ðŁı ¹ +Ë Ĵ +ðŁ¤ ± +ãı ľ +ðŁİ Į +ï¥ Ń +ঠ£ +ðŁİ ¹ +ãĬ Ł +à´ ° +ðĿIJ Ķ +à´ ¨ +འļ +âľ º +Õ · +ðŁij ³ +ঠľ +âĺ ĭ +âĻ Ĭ +ãĢ Ľ +È ĭ +à® ° +áĥ ¨ +âĦ ķ +íij Ģ +ðĿĵ ĥ +ðŁ¦ Ķ +Ä ¿ +Å Ģ +Æ ³ +É ļ +Ö ĥ +Ü £ +ß Ł +à¦ Ń +ৠ¡ +ච» +ຠ£ +འĩ +Ḡ¨ +á½ Ī +â½ ¬ +ê¡ Ķ +ì³ Ħ +ï¨ ī +ðĿIJ ¡ +ðĿĺ ¢ +ðŁį ¿ +ðŁİ Ł +ðŁı ī +ðŁĶ IJ +ðŁļ ħ +ðŁ¤ ½ +Æ į +Ç « +Ç ½ +È ļ +Î ī +Ó ¤ +Ó ª +Õ Ĭ +Ù ¼ +Ú ´ +ß Ŀ +චľ +á¼ ķ +á¿ ¥ +âİ ŀ +ãĢ ļ +ãī ¤ +ê³ ¸ +ê· ģ +ëĵ Ħ +ëĵ ķ +ì¨ Ķ +ì± ¨ +ðĿIJ ¾ +ðĿij » +ðĿĶ ¼ +ðĿķ Ŀ +ðĿĺ Ń +ðŁĨ Ļ +ðŁĵ ¤ +ðŁĶ Ł +ðŁĹ ¼ +Ä ľ +Æ ģ +Æ ¿ +Ç ³ +Ç · +É ĥ +É ł +Ê ī +Ê § +Ë ² +Ï ´ +Õ ģ +Õ ŀ +Ö ĩ +Û Ĥ +Û ĵ +ß Ĺ +ß ¦ +ঠ¹ +à® ³ +à´ ¸ +à» Ĥ +áĪ Ŀ +áĪ ª +áĭ µ +áIJ Ĭ +áĴ ª +áļ ĸ +áŀ Ľ +á´ ¢ +áµ ı +áµ Ń +ᶠ« +Ḡı +ẠĴ +á¼ ¥ +á½ ķ +á½ ¼ +âĤ Ĭ +âĦ Ĥ +âĦ © +âĩ ī +âī £ +âĮ ł +âİ Ł +âı ® +âķ ĺ +âĹ ĸ +âĺ © +âĻ ij +âĻ ² +âļ Ľ +ãĦ Ł +ãī ± +ãİ ļ +ê¡ ķ +êª ĸ +ê° ¹ +ê² Ĩ +êµ Ħ +ëĩ ¬ +ëĭ ¯ +ëı ł +ëĴ ¬ +ëĸ Ī +ëĸ ½ +ëĺ Ķ +ëŀ ¸ +ë¸ ħ +ë» ł +ë¿ Ł +ìĤ µ +ìĬ ī +ìľ ° +ìł ĭ +ìł Ķ +ì¥ ¡ +ìŃ Ŀ +ì¼ ¬ +íĪ ĩ +íī ľ +íį Ħ +íĽ ¾ +íĿ £ +ï¤ © +ï¤ ¯ +ï¦ ľ +ï¦ § +ï§ ľ +ï¨ Ī +ï¬ ª +ï ¬´ +ïŃ ½ +ï® ī +ï¯ ŀ +ï° Ĵ +ï± ĩ +ï¿ Ħ +ðĿIJ ħ +ðĿij Ħ +ðĿij º +ðĿĴ Ĺ +ðĿĵ ® +ðĿķ Ľ +ðĿķ ŀ +ðĿĸ ij +ðĿĺ ģ +ðĿĺ Ĩ +ðĿĺ ¶ +ðĿĻ ¢ +ðĿļ ľ +ðŁĮ ĥ +ðŁĮ ¦ +ðŁį Ł +ðŁİ İ +ðŁı Ļ +ðŁIJ © +ðŁIJ « +ðŁIJ ´ +ðŁij Ķ +ðŁĵ ī +ðŁĵ Ľ +ðŁĶ ī +ðŁĸ ¼ +ðŁĹ ĥ +ðŁĹ ¯ +ðŁļ ĩ +ðŁļ IJ +ðŁļ µ +ðŁ¤ ¶ +ðŁ¥ ĭ +ðŁ¥ ĵ +ðŁ¥ ® +ðŁ¦ İ +ðŁ¦ ł +ðŁ§ Ĵ +ðŁ§ ¨ +Æ IJ +Ç į +Ó Ģ +Ô Ľ +ಠ° +à´ Ļ +áĢ Ĵ +ê² Ŀ +ê¹ ¹ +ë© ¥ +ìĸ Ķ +ï¤ ģ +ï¤ ı +ï¦ ī +ï¦ ĵ +ï§ ī +ï² Ŀ +ðĿĹ ŀ +ðĿĹ ± +ðŁĮ ĭ +ðŁį ¶ +ঠļ +ìķ ľ +ðĿIJ ¯ +ðĿļ Ŀ +à° ¨ +འĺ +འł +á¡ ¥ +á¾ ° +âģ į +âĶ ° +⬠ľ +ðĿIJ ł +ðĿij ¯ +ðĿĹ Ľ +ðĿĵ » +ðĿĸ Ī +âŀ » +áŀ ł +â¡ ± +â» ij +ðŁ§ µ +ï¦ ¢ +ðŁij ĺ +ãĤ Ķ +â¼ Ł +ãĬ ¤ +ï¦ Ŀ +ãĮ ¦ +âĢ ¸ +ðŁĶ Ļ +ã ¹ +ã¹ ¦ +ï¹ ħ +ï© Į +ãī ¨ +ï¸ ½ +âį ¥ +ðŁļ ī +ðŁ¥ ľ +âĵ ľ +â» Ŀ +ï¨ ľ +ðŁĴ Ĵ +áĦ ij +â¾ ŀ +ï¨ ģ +à´ ª +áĦ İ +âŀ ´ +ঠ· +áħ ¬ +áŀ § +âĨ ¢ +âķ ¦ +âľ ij +Ë ¬ +Õ IJ +༠Ķ +Ê ¤ +Ë ¨ +ठŀ +à» ĥ +༠ļ +âĵ ¥ +âķ ľ +ðŁIJ ĸ +á¼ Ļ +á¼ ¤ +ìĨ ° +È Ĥ +Ê ± +à® ļ +áĥ § +á´ ĭ +á´ ® +âĿ ¡ +âŀ · +ëĿ ¡ +ï§ ¢ +ï¯ ¡ +ðĿķ ķ +ðŁħ ° +ðŁ¦ ¸ +Ç ¸ +Ó ŀ +Ô ¶ +Ö Ĩ +Ú ģ +Û ĭ +áİ ¥ +á¾ ¿ +âĶ Ń +âĶ ® +êĢ Ģ +ê± ĺ +ëIJ Ń +ë½ Ħ +ìĶ IJ +ì¸ Į +íģ ł +íĻ ± +ï¥ ī +ï¨ ĸ +ðĿij ´ +ðĿĸ Ĵ +ðĿĺ ¨ +ðĿ ļĮ +ðŁIJ ¡ +ðŁij ¢ +ðŁĵ Ķ +Å ħ +Æ İ +È © +Ò ª +Ô ĥ +áĥ « +Ḡĩ +⼠Ł +ê» Ń +ë¨ Ħ +ìŁ Ģ +ì¤ ´ +íļ IJ +ï¤ ³ +ðŁŁ ¢ +Æ § +È ¼ +Ê Ŀ +Ë Ħ +Ë ħ +Ë į +Ë § +Ò ¥ +Õ Ķ +Ø ı +Ø ¼ +ß IJ +ß ľ +ठĵ +ঠĻ +à® ĵ +ච´ +༠į +༠Ĵ +འ£ +áĢ Ĥ +áĢ Ĭ +áĦ Ħ +á Īĺ +áĭ Ĭ +áĮ į +áij ĭ +áŀ Ĥ +áł ¢ +á¡ Ŀ +á´ ¦ +áµ į +áµ ¨ +Ḡ¡ +Ḡ¯ +á¼ £ +âģ Ĥ +âĦ ĺ +âĦ ľ +âĦ ³ +âĦ µ +âĨ ¦ +âĩ Ĩ +âĪ · +âĬ ļ +âĮ « +âĮ ¯ +âİ Ľ +âİ ľ +âİ ¤ +âİ ¦ +âİ ® +âij ī +âĶ ī +âķ Ļ +âĸ Ĥ +âĹ Ń +âĺ Ĭ +âĺ į +âĺ Ĵ +âļ Ĩ +⼠§ +⼠² +âŀ ĺ +⥠Ħ +â´ ³ +â´ ½ +âµ Ī +ãī ¯ +ãİ ij +㧠¬ +êĻ ¬ +ê§ ģ +ê³ ¬ +ê´ ŀ +ê» ľ +ëħ ĵ +ëĭ ¼ +ëį ĸ +ëĸ ± +ëĿ ° +ë¡ ¹ +ë¢ ´ +ë£ Ģ +ë¤ ł +ë¨ ķ +ëŃ ¥ +ìĦ ¶ +ìħ ¤ +ìĮ ķ +ìį ª +ìı © +ìĴ Ģ +ìĶ ¯ +ìĿ Ķ +ìĿ ľ +ìł Ń +ì§ ¦ +ì¨ © +ì² ¬ +ì³ ¥ +ì¼ ¯ +íĢ « +íĢ Ń +íĥ ¸ +íĵ ģ +íķ ¬ +íĹ ¸ +íĽ ķ +íľ Ń +íĿ Ĺ +ï¤ Į +ï¤ ª +ï§ ¿ +ï¬ Ħ +ï¬ ħ +ïŃ ij +ïŃ « +ïŃ º +ï® Ĥ +ï® ¢ +ï® ¨ +ï° İ +ï° ł +ï² £ +ï³ IJ +ï³ Ĵ +ï³ ĺ +ï³ ľ +ï¹ ¼ +ï¿ ¨ +ðĿIJ © +ðĿĴ ļ +ðĿķ Ķ +ðĿķ ¤ +ðĿĸ Į +ðĿĹ £ +ðĿĹ ° +ðĿĹ ´ +ðĿĺ Ĥ +ðĿĺ ¥ +ðĿĺ ® +ðĿĺ ¸ +ðĿĻ Ģ +ðĿĽ ¾ +ðĿľ ı +ðŁĮ ģ +ðŁĮ ľ +ðŁĮ ¥ +ðŁĮ ¯ +ðŁį IJ +ðŁİ Ĵ +ðŁı Ķ +ðŁı ķ +ðŁı ® +ðŁIJ Ĥ +ðŁIJ ī +ðŁIJ ¹ +ðŁĶ ķ +ðŁĶ ļ +ðŁķ ij +ðŁķ £ +ðŁĹ ŀ +ðŁĹ ¡ +ðŁĹ ¿ +ðŁļ Ĩ +ðŁļ Ĭ +ðŁļ ĵ +ðŁļ ķ +ðŁļ ¾ +ðŁĽ ģ +ðŁĽ İ +ðŁĽ ı +ðŁ¤ ´ +ðŁ¥ ķ +ðŁ¥ ĸ +ðŁ¥ ł +ðŁ¥ ¥ +ðŁ¦ Ĩ +ðŁ¦ ī +ðŁ¦ ļ +ðŁ§ ij +ðŁ§ ¥ +ðŁ§ ¿ +Å ° +Æ º +É § +ઠĩ +à® £ +áĪ Ī +áĬ ¤ +áĭ ® +áĮ Ī +áĮ µ +ᥠ² +âĵ Ł +êĻ ³ +ê° Ĭ +ëķ ģ +ëķ ¨ +ìĬ ģ +ï¦ µ +ï¬ ² +ðĿĸ į +ðĿĺ Į +ðĿĺ ³ +ðĿĻ © +ðŁį Ļ +ðŁĸ ĸ +áī ³ +áĭ ¨ +áĸ ĩ +áŀ Į +á¹ § +âķ ª +âŀ ļ +â² ĺ +ê ķ +êķ ¥ +ï¤ · +ï® £ +ï¯ ł +ðĿĴ ĸ +ðĿķ ĺ +ðĿĸ ĩ +ðĿĹ Ł +ðĿĹ ª +ðĿĹ ¯ +ðĿĻ ł +ðŁĵ ı +à¦ Ĺ +âĴ » +â² ł +ðĿĵ µ +Ê £ +à° ľ +áĬ ¢ +áŀ IJ +Ḡ· +âĦ Ľ +âĩ Ģ +âĩ Ĭ +êĴ ¦ +ê¦ ł +ï® ¤ +ðŁį Ľ +ðŁ¤ Ľ +ᨠ¾ +âŀ º +áķ ¯ +ἠı +âĩ Ĥ +âĶ ¹ +âĻ Ĺ +ðŁĸ ¨ +ê¦ ı +ઠ° +áļ ¨ +ðŁ¤ ¥ +ðŁ§ ¢ +ãIJ Ĥ +ãĦ ¥ +ðŁĸ Į +â¼ Ĵ +ãĬ § +âį © +ðŁ¦ ij +âĶ · +ï© IJ +ï© ¡ +ðĵ Ī +ðĵĪ Ĵ +â» Ħ +ï¨ Ĵ +âĦ ª +Ò § +Ú Į +âĢ ¶ +⺠ł +â» ģ +âĨ ¸ +áĦ IJ +ãħ IJ +à» Ħ +áĹ ª +âĨ ¼ +âĩ ĭ +âĩ ĺ +âĮ ij +âĸ © +ðĿIJ Ĺ +Ä Ĭ +ঠī +ìī ł +É ¤ +ß į +ß ı +áµ Ĺ +âĤ ¥ +âĵ ī +âĶ ł +âĶ ¨ +âķ Ħ +ä ¤ +ä¤ Ģ +ê» ¸ +ï® ģ +ðĵ Ĥ +ðĵĤ ĥ +ðŁ¦ ķ +Æ Ľ +ঠĩ +ãı ĺ +ï® ¼ +Ú ĵ +Ú Ŀ +ঠĵ +ච¯ +á´ ħ +á½ Ļ +âģ ¼ +âĸ İ +â¼ © +ä Ķ +äĶ Ģ +ë» ¡ +ìĽ ½ +íģ Ħ +ï¥ ¼ +ï± ī +ï¹ » +ðĿĸ ĭ +ðĿĻ Ī +ðĿĻ ª +ðĿ Ļ¶ +ðŁIJ Ħ +ðŁIJ Ĩ +áİ ¢ +ḠĮ +âĿ ´ +ðŁı ¸ +È Ŀ +É ¸ +Î ħ +Ï ľ +Ó ¢ +Õ ¹ +à´ ħ +ຠĪ +áĭ ° +áij İ +áł µ +á¡ ł +á´ ī +Ḡµ +á¿ ´ +âĵ £ +âĶ ¶ +â½ ¯ +ê² ¥ +ê¿ ĺ +ëģ İ +ëİ Ī +ëĶ ¯ +ë² ° +ìĺ ¯ +ìĽ ¸ +ìŀ Ĺ +ì§ ĺ +ì¬ ¬ +ì· ¬ +íģ ħ +íĵ Ķ +íĽ Ŀ +ï¤ ® +ï¤ ¹ +ï¥ ² +ï¯ ĸ +ðĿĵ ħ +ðĿĻ Ħ +ðŁĵ ¶ +ðŁĹ Ĵ +ðŁ¥ Ķ +ðŁ¥ Ń +Å ® +Å ´ +Æ ī +Æ « +Ç ģ +Ç £ +Ç º +Ç ¼ +È į +È ¯ +É ľ +Ê ¬ +Ë ģ +Ë ¤ +Ë µ +Ï Ľ +Ò ¤ +Ò ¬ +Ó ı +Ó Ľ +Ó ¡ +Ó ³ +Ô Į +Ô ¬ +Õ ³ +Ù » +Ú ī +Ú § +Ü ľ +ß ª +ठĿ +ঠĽ +ਠĨ +ઠķ +ઠ¡ +à® İ +à° ¬ +ൠ» +ൠ¼ +චł +à¶ Ń +ච¶ +à· Ĩ +༠½ +áĢ ļ +áħ ¢ +áĨ ¸ +áĪ Ģ +áĪ ķ +áĪ ° +áī ¡ +áī ¤ +áĬ ¦ +áĬ « +áĭ ĭ +áĭ į +áİ ¯ +áij Ń +áķ Ĺ +ᣠĽ +ᥠĴ +á© ī +áŃ º +á´ ¡ +áµ ĺ +áµ Ľ +ᶠł +Ḡģ +Ḡĭ +á¹ Ļ +á¹ Ŀ +á¹ ¦ +Ạħ +á¼ Ĥ +á½ ĥ +á½ į +á½ § +á¾ · +âĢ µ +âĤ İ +âĦ Ŀ +âħ Ģ +âĨ ŀ +âĨ § +âĩ ħ +âĪ ĥ +âī ı +âī ½ +âĬ ŀ +âĬ ¡ +âĬ § +â Ĭ¶ +âĭ Ħ +âİ Ĵ +âİ ¡ +âİ £ +âİ ª +âı İ +âĵ ĥ +âĵ ĸ +âĵ ¨ +âķ ĭ +âķ ĸ +âķ ¢ +âķ ² +âĸ Ĩ +âĸ Ĭ +âĸ į +âĸ ® +âĺ ¡ +âĺ ¦ +âĺ ± +âĺ ¿ +âĻ ĺ +âĻ Ŀ +âļ ° +⼠ij +âŀ ª +⤠Ŀ +⤠¢ +⤠· +⧠« +â¨ Ń +⨠¯ +â± £ +â² İ +âµ Ľ +ãħ Ķ +ãĪ ı +ãī ² +ãī ³ +ãĬ ij +ãĭ Ľ +ãİ IJ +ê² ¤ +ê· ¿ +ê¹ ŀ +ê» ¨ +ê¼ į +ê¿ ¸ +ëĥ ¬ +ëĩ IJ +ëĭ ł +ëį ¯ +ëĹ Į +ëĹ ij +ë¥ Ģ +ëª ĥ +ëª ¯ +ë± ¡ +ë³ ĵ +ë³ ½ +ë µľ +ìĤ ³ +ìħ ¥ +ìĩ ½ +ìı ¨ +ìı ¸ +ìķ į +ìĸ ĸ +ìŁ ¨ +ì¢ ĥ +ì¢ į +ì¥ ij +ì§ ¼ +ì© ĥ +ì® ľ +ì® ¸ +ì³ ij +ì´ ¥ +ì¾ ĥ +íħ ¦ +íĪ ¿ +íĵ ½ +íķ ³ +íĸ ı +íĹ ł +íĿ « +ï¤ ĵ +ï¤ ĺ +ï¥ İ +ï¥ ¶ +ï¦ ħ +ï¦ ½ +ï§ ĩ +ï¬ Ĩ +ï¬ ³ +ï® ĩ +ï® Ī +ï® Ŀ +ï® © +ï® ± +ï¯ ĺ +ï¯ Ļ +ï¯ ¢ +ï¯ £ +ï¯ ¤ +ï¯ ¥ +ï± Ĥ +ï² Ĩ +ï² ª +ï´ ¼ +ïº ī +ïº Ĭ +ïº ¥ +ðĿij ¨ +ðĿij © +ðĿij ² +ðĿ ĴĮ +ðĿĴ ª +ðĿĴ ® +ðĿĵ Ĥ +ðĿĵ Ī +ðĿĵ ¯ +ðĿĶ ¨ +ðĿķ Ģ +ðĿķ Ĩ +ðĿķ ¦ +ðĿķ § +ðĿķ « +ðĿķ · +ðĿĹ µ +ðĿĹ ¸ +ðĿĺ Ħ +ðĿĺ Ļ +ðĿĺ ł +ðĿĺ ¬ +ðĿĻ į +ðĿĻ ij +ðĿĻ ¡ +ðĿ Ļ¨ +ðĿĻ · +ðĿļ į +ðĿĽ ¿ +ðŁ ĥ +ðŁĥ ı +ðŁħ ĺ +ðŁ ī +ðŁī ij +ðŁİ ¡ +ðŁİ ª +ðŁİ ± +ðŁİ ³ +ðŁİ º +ðŁı İ +ðŁı Ĺ +ðŁı ļ +ðŁı ŀ +ðŁı ¦ +ðŁı § +ðŁIJ ģ +ðŁIJ ħ +ðŁIJ ĵ +ðŁĴ Ĥ +ðŁĵ ij +ðŁĵ ĵ +ðŁĵ ¨ +ðŁĵ « +ðŁĶ ĭ +ðŁĶ Ń +ðŁĶ ¯ +ðŁķ Ĺ +ðŁļ Ĥ +ðŁļ ¢ +ðŁļ ¦ +ðŁļ ¬ +ðŁĽ ĭ +ðŁĽ Į +ðŁĽ ¬ +ðŁĽ ¶ +ðŁŁ ¡ +ðŁ¥ ĺ +ðŁ¥ Ł +ðŁ¥ ¦ +ðŁ¦ ĩ +ðŁ¦ Ī +ðŁ§ Ĭ +ðŁ§ Ĺ +ðŁ§ ¤ +Ê · +Ë ¹ +á¹ ļ +á½ ¥ +âĦ Ł +ê² ¯ +ê» « +ë° · +ìĥ Ĩ +ìĽ Ŀ +ì¨ ī +ì« ı +ï¯ ķ +ðĿľ ĭ +É ² +Ò Ń +Ó Ī +འĽ +áĭ ĵ +áĻ Ń +áł © +á¹ ® +âĦ Ĵ +âĨ » +âµ ĥ +ëĢ ¨ +ëł § +ìī ¥ +ìĮ ľ +ìĹ ¶ +ì¨ Ī +ìª ¾ +íı ½ +íļ Ķ +íĽ µ +ï¤ ¸ +ï¦ IJ +ï§ Ĺ +ï§ ļ +ï¬ ¯ +ðĿIJ Ĭ +ðĿķ Ĺ +ðĿĹ ļ +ðĿļ ĸ +ðŁħ ´ +È ĥ +É Ŀ +Ï ± +Ó Ĺ +ठ¢ +áħ ł +áī ¦ +áij Į +áĴ ¼ +áŀ ¡ +áł ¨ +áł Ń +ᨠħ +ᨠĶ +á´ ĺ +ᶠ¦ +á¸ İ +á¼ ħ +á¼ ¹ +âĨ ¯ +âĵ İ +ãı Į +ê ī +êī Ĥ +ëĨ § +ëĿ ± +ì¢ ¡ +íĪ ½ +ï¤ ĩ +ï¤ Ľ +ðĿIJ ķ +ðĿĵ ¸ +ðĿĵ ¼ +ðĿĹ ķ +ðĿĺ Ī +ðŁı £ +ðŁı ¤ +ðŁĹ Ħ +Ñ · +Ò ł +áµ ĸ +á¼ ¨ +ë¬ Ħ +ï° ´ +âĪ ½ +Õ Ń +Ú ¹ +ॠŁ +áĢ Ĩ +áŀ Ĵ +ãĢ ¶ +ê¦ « +ï¸ ĵ +ðĿIJ Ľ +ðĿĺ Ĺ +ðŁı ľ +ì« Ń +ðŁ§ ŀ +འĤ +âĨ ¿ +âĩ ı +âĵ ģ +âĶ § +âķ ģ +âķ ¤ +ê¦ Ĺ +ê¦ ¤ +ðŁı Ī +áŀ ķ +Ô ½ +àª Ĺ +ଠĨ +âķ ķ +ï½ ł +â¼ ¦ +â¼ ¯ +â¾ · +âĶ ĸ +ଠĵ +âĺ Ĺ +âį ĭ +ï¨ Ŀ +â¼ ¥ +ï¦ ª +âĦ Ĭ +ãĢ ´ +âį ¢ +ð¡ Ī +ð¡Ī ½ +ï© ¨ +ãĢ » +ãı ĥ +ï¦ ¡ +ï¨ ĺ +ðŁIJ ĥ +ðŁĨ ĸ +ðŁĹ ¾ +ãĦ ĩ +Þ ĭ +â¼ ¼ +ï¨ Ń +Þ Ģ +Þ Ħ +Þ Ī +Þ IJ +âĮ Ħ +â» ĺ +ãŁ ¢ +á ħ§ +ðIJĮ ¿ +Ë » +à² Ĺ +áĢ ĩ +áŀ Ĭ +âķ ĩ +ãĩ ¼ +ãİ ° +Õ Ĵ +Ü Ī +ß ¥ +à¿ IJ +áĢ Ł +âĨ ¥ +âķ Į +â½ Ģ +â½ ° +â¾ Ĭ +ä Ħ +äĦ Ģ +ðĵ IJ +ðĵIJ į +ðŁİ ¦ +âĤ ¯ +âĬ ĺ +âĦ į +Ê µ +Ñ ¶ +Ú ĥ +ঠĶ +à´ ¦ +áİ ¶ +áĵ ķ +á¹ ¨ +âĤ ł +âĩ ° +âĹ Ĵ +â¿ Ĭ +ê· ± +ì¹ ķ +íĪ © +ïŃ Ģ +ðĿĴ ¸ +ðĿĵ Ĭ +ðĿĺ © +Ç ¦ +É « +áĬ ¨ +È ¹ +Ê ¯ +Î ª +Ú Ģ +áĮ ¸ +áİ » +áı ķ +áı ´ +á² Ĥ +á½ ¨ +âı Ŀ +âĺ Ļ +ëĥ ¨ +ëĦ ¼ +ëĪ Ļ +ë£ ħ +ìĶ ¼ +ìķ Ŀ +ìļ ¬ +ìľ ± +ï¥ Ĥ +ï¦ ¹ +ï¬ ¹ +ïŃ ģ +ï³ Ī +ðĿĶ ħ +ðĿĺ ¤ +ðĿĻ ı +ðĿĻ Ļ +ðŁķ ī +ðŁ§ Ļ +Ḡij +ê´ ¼ +ëģ į +ëĹ ´ +ëĿ ³ +ë° ŀ +ë° ¢ +ëµ ĺ +ìĤ Ķ +ìĦ Ħ +ì¼ ļ +íĢ ł +íĬ ± +íĮ ĸ +ï¤ ij +ï¦ ´ +ï¦ ¸ +ï´ į +ðĿĺ · +Ä ¬ +Å ¬ +Æ Ģ +Æ ĭ +Æ ľ +Ç ij +Ç ĺ +Ç ŀ +Ç ¥ +Ç ® +É ° +É ¶ +É · +É ½ +Ê Ī +Ê IJ +Ë İ +Ë Ł +Ë ¦ +Ë ¯ +Ï IJ +Ï ĵ +Ï ¢ +Ï ¤ +Ï ª +Ï Ń +Ï ® +Ï » +Ñ ł +Ñ Ń +Ò ¨ +Ó Ŀ +Ô ¡ +Ô · +Õ ī +Õ ĵ +Õ ĸ +Õ ļ +Õ Ŀ +Ö İ +Ø ¿ +Ú ħ +Ú į +Ú Ķ +Û Ĭ +Û ¾ +Ü Ļ +Ý Ĵ +Ý ĺ +ß Ĵ +ß ĸ +ठĬ +ठIJ +ঠı +ঠĸ +ৠŁ +ઠ® +ઠ¹ +à® ħ +à® Ĩ +à° ¡ +à° ° +ಠļ +ಠ® +ಠ¯ +à´ Ł +à´ · +ൠ¾ +චij +චŀ +༠¼ +འĵ +áĢ ĵ +áĤ ¦ +áĥ ĸ +áĥ Ń +áĥ ¯ +áħ ¨ +áħ ª +áĨ ° +áĪ ģ +áĪ İ +áĪ ĵ +áĪ ¥ +áĪ ² +áĪ ´ +áĪ » +áī ł +áī ² +áī ¶ +áĬ £ +áĬ ¥ +áĬ ª +áĭ ĺ +áĭ ² +áĭ ¶ +áĮ £ +áį ¡ +áį £ +áİ ¬ +áİ ¾ +áIJ ¡ +áķ ķ +áĸ ± +áĹ IJ +áĹ Ń +áĺ ī +áļ ± +ἠŁ +áŀ ¥ +ᣠĶ +áł £ +áł ª +áł ° +áł ´ +ᤠĸ +ᥠ£ +á ® +á® ł +á ¯ +ᯠĻ +á ° +á° į +á´ Ĭ +á´ ¾ +áµ ģ +áµ İ +áµ ŀ +áµ ¤ +ᶠħ +ᶠĺ +ᶠŁ +ᶠ¢ +ᶠ¤ +ᶠ± +ᶠ» +Ḡī +Ḡŀ +Ḡº +á¹ ĵ +á¹ Ĺ +á¹ ª +ẠĬ +Ạı +ẠĽ +á¼ ĥ +á¼ Į +á¼ ¿ +á½ Ĥ +á½ ĵ +á½ Ĺ +á½ ¦ +á¾ ± +á¾ ´ +á¿ ĺ +á¿ Ł +á¿ ¸ +âģ ĺ +âĤ ij +âĤ Ľ +âĤ ¿ +âĦ ĩ +âĦ ŀ +âĦ ± +âĩ Ł +âĩ ² +âĪ ¤ +âĪ ¶ +âī Ĥ +âī ¾ +âĬ ¨ +âĬ ³ +âĬ · +âĭ Į +âĭ ĺ +âĮ ķ +âĮ ¥ +âĮ µ +âĮ º +âį £ +âį ² +âį µ +âİ ĩ +âı ĥ +âı IJ +âı ł +âı ¤ +âı ¶ +âı ¸ +âı ¹ +âij Ĥ +âĴ · +âĴ º +âĵ ¡ +âĵ ¤ +âĶ ¾ +âĸ ĺ +âĸ µ +âĹ ª +âĹ · +âĺ ¨ +âĺ « +âĺ ² +âĺ ³ +âĻ Ĩ +âļ ¤ +âļ ¥ +⼠ĵ +⼠´ +⼠¾ +âŀ « +âŀ ¿ +⣠· +⤠ij +⤠« +⤠¶ +⤠½ +⧠ª +⨠Ģ +â ©½ +⬠¡ +⬠¢ +⬠¤ +â² ĸ +â² ª +âµ Ģ +⸠® +⸠½ +ãĢ ł +ãĢ · +ãĦ Į +ãĦ ĺ +ãħ ij +ãĪ İ +ãĪ IJ +ãĬ ľ +ãĮ ĵ +ãĮ ł +ãİ Ł +ãİ ¤ +ãİ § +㬠® +ä Ī +äĪ Ģ +ä ° +ä° Ģ +ê ħ +êħ ī +êĩ Ĺ +ê Ī +êĪ į +ê§ Ĥ +ê§ Ĭ +êª Ģ +ê² Ī +ê² į +ê³ Ģ +êµ ł +ê½ IJ +ê¾ Ī +ê¿ ± +ëĥ ı +ëĦ ij +ëħ ¤ +ëĩ ¸ +ëĪ ¼ +ëī ħ +ëĬ £ +ëĭ º +ëį ŀ +ëIJ Į +ëķ ¸ +ëĺ ł +ëĻ ĩ +ëĻ Ī +ëľ ½ +ëŀ Ķ +ëł ľ +ë£ IJ +ë§ Ģ +ë§ Ĭ +ëª Ģ +ë¬ Ń +ë¯ ¾ +ë³ ľ +ë´ Ĭ +ëµ ī +ë· ľ +ë¸ Ģ +ë¹ ĭ +ìģ Ħ +ìĤ £ +ìĤ » +ìĦ µ +ìħ Ĵ +ìī Ī +ìī Ķ +ìĬ Į +ìĬ Ļ +ìIJ ´ +ìĵ º +ìķ ļ +ìķ º +ìĸ ľ +ìĹ ª +ìĺ ľ +ìĻ ¤ +ìļ Ľ +ìļ º +ìĿ ħ +ìĿ ı +ìĿ Ń +ìĿ ¶ +ìł Ľ +ì¡ Ī +ì¢ ī +ì¢ Ķ +ì© ł +ìŃ Į +ì¯ © +ì´ £ +ì¸ ķ +ì¹ Ł +ì¾ ¡ +ì¿ Ļ +íģ ĩ +íģ ī +íĩ Ģ +íĪ ¶ +íĸ ij +íĸ ¤ +íĹ ħ +íľ ı +íĿ Ŀ +ï¤ Ĵ +ï¤ ķ +ï¤ ¬ +ï¥ ħ +ï¥ ĩ +ï¥ ı +ï¥ ļ +ï¥ Ł +ï¦ Ħ +ï¦ Ī +ï¦ ¨ +ï¦ © +ï¦ ² +ï§ ģ +ï§ ĥ +ï§ Ķ +ï§ ł +ï§ £ +ï§ ® +ï ŃIJ +ïŃ ĸ +ïŃ ¦ +ïŃ ´ +ïŃ µ +ïŃ ¶ +ïŃ ¸ +ï® Į +ï® İ +ï® ŀ +ï® Ł +ï® ¡ +ï® ª +ï¯ Ķ +ï¯ Ĺ +ï¯ ļ +ï¯ Ľ +ï¯ Ŀ +ï¯ Ł +ï¯ § +ï¯ ¨ +ï¯ « +ï¯ ¯ +ï¯ ° +ï¯ ± +ï¯ ² +ï¯ ³ +ï¯ ´ +ï¯ µ +ï¯ ¶ +ï° Ģ +ï± ħ +ï± Ķ +ï± ´ +ï² ģ +ï³ ķ +ï· ½ +ï¸ ķ +ï¸ ± +ï¹ £ +ï¹ ½ +ï» į +ï¾ ± +ðĿIJ Ļ +ðĿIJ ½ +ðĿij ¤ +ðĿij ® +ðĿij µ +ðĿĴ ĥ +ðĿĴ Ħ +ðĿĵ Ń +ðĿĵ · +ðĿĶ ĸ +ðĿĶ ŀ +ðĿĶ ¢ +ðĿĶ ¦ +ðĿĶ ¬ +ðĿķ Ħ +ðĿķ Ĭ +ðĿķ İ +ðĿķ Ļ +ðĿķ ľ +ðĿķ Ń +ðĿķ ³ +ðĿķ ¸ +ðĿķ ¾ +ðĿ ĸī +ðĿĸ ı +ðĿĺ ĩ +ðĿĺ ī +ðĿĺ ĸ +ðĿĺ Ľ +ðĿĺ ŀ +ðĿĺ « +ðĿĺ ¾ +ðĿĻ ĩ +ðĿĻ ī +ðĿĻ ĭ +ðĿĻ İ +ðĿĻ ĺ +ðĿĻ ¥ +ðĿļ ĥ +ðĿļ IJ +ðĿļ Ķ +ðĿľ ĥ +ðŁĦ · +ðŁħ Ŀ +ðŁħ ¾ +ðŁĨ Ĥ +ðŁĨ ĵ +ðŁĮ Ĥ +ðŁĮ Ĩ +ðŁĮ ī +ðŁĮ ij +ðŁĮ ĺ +ðŁĮ © +ðŁĮ « +ðŁį ¢ +ðŁį ¥ +ðŁİ Ľ +ðŁİ ¢ +ðŁİ ´ +ðŁij ¡ +ðŁĴ ¾ +ðŁĵ Ń +ðŁĶ Ī +ðŁĶ ¦ +ðŁĶ ² +ðŁĶ ³ +ðŁķ ĵ +ðŁķ ķ +ðŁķ ĺ +ðŁķ Ł +ðŁķ · +ðŁĹ ³ +ðŁļ Ħ +ðŁļ Ķ +ðŁļ ĸ +ðŁĽ IJ +ðŁĽ ¤ +ðŁĽ ¸ +ðŁ ł +ðŁł ³ +ðŁ¤ ¹ +ðŁ¥ ĥ +ðŁ¥ ¨ +ðŁ¥ ª +ðŁ¥ ¾ +ðŁ¦ ĥ +ðŁ¦ Ĵ +ðŁ¦ Ļ +ðŁ¦ ¶ +ðŁ§ ł +ðŁ§ ª +ðŁ§ Ń +ðŁ§ ² +𣠷 +ð£· Ń +ð¦ ĺ +ð¦ĺ Ĵ +Æ ij +Ç Ļ +È ® +Ø ł +Ú Ħ +Ü Ģ +ß ¢ +áī Ģ +áĬ IJ +áİ ł +Ạŀ +ëĪ ŀ +ëķ Ł +ë£ ģ +ë¤ Ĺ +ìĦ ¥ +ìħ ij +ìĸ IJ +ìĽ Ľ +ì£ ķ +íİ ı +íĽ ĵ +ï¥ º +ï³ Ľ +ï´ « +ðĸ § +ðĸ§ · +ðĿķ ģ +ðŁIJ ª +ðŁĴ Ī +ðŁĵ ł +ðŁķ Ľ +ðŁķ ´ +Ñ Ŀ +Ó Ĭ +ॠ² +ઠª +áĥ ¤ +áį IJ +ᶠ° +á¼ Ŀ +á½ © +âĭ ĭ +âĴ ½ +âĻ ¾ +â ½Ķ +â¾ ¯ +ãĦ Ĵ +ãħ ļ +ëIJ į +ë· ģ +ìĭ Ģ +ìļ Ŀ +ì¥ ° +ìº ´ +íĭ ī +íĿ ½ +ï¦ Ģ +ï¦ ¿ +ï§ ħ +ï§ ĵ +ïŃ ¯ +ï® Ĩ +ðIJ¤ ķ +ðĿIJ Ł +ðĿĴ ħ +ðĿĵ ľ +ðĿĶ ° +ðĿĶ » +ðĿĺ į +ðĿĻ ¯ +ðŁĦ ½ +ðŁħ Ĥ +ðŁħ Ķ +ðŁħ ½ +ðŁĵ ´ +ðŁ§ ĸ +Ó Ĵ +Ḡ² +ëī ¼ +Ç ı +È ĵ +Ê ¸ +Õ Ĥ +Û ħ +ß ¡ +ß £ +à® ¯ +à° Ī +ಠ¸ +ຠ® +༠ķ +áĢ İ +áĨ ¡ +áIJ ĭ +áIJ ķ +áij ¯ +áŀ Ĩ +ᨠķ +á© Ī +âģ ħ +âĨ ļ +âĶ İ +âł © +â² Ĥ +â² Ķ +â² ¨ +ãĬ ļ +íĵ ² +ðĿij Ī +ðĿij ¬ +ðĿij ¹ +ðĿĴ ¾ +ðĿĵ ± +ðĿĵ ½ +ðĿķ ¯ +ðĿķ » +ðĿĺ ½ +ðĿļ Ĩ +ðŁĦ ° +ðŁIJ ¨ +Ò ķ +ಠħ +ï¨ Ĩ +ðĿij ° +ðŁĦ ¸ +Ô İ +Ø į +Ù µ +ಠ¶ +áĢ Ī +áĺ Ĺ +áł ¸ +á¡ ¡ +ᨠ² +á© ģ +á´ · +áµ § +âķ ¨ +âļ ģ +â¾ Ŀ +ãĢ ¼ +ãĦ ı +êĴ « +ê¦ ¥ +ê¦ © +ê¦ ² +ìĺ ¼ +íĵ IJ +ðĵ ĩ +ðĵĩ ¼ +ðĿķ ¿ +ðŁĽ ´ +ë¨ ľ +ಠµ +à´ İ +༠Ģ +âĩ ĸ +ãĪ « +âĵ Ģ +áħ ´ +áļ ¾ +ἠŀ +ἠ« +ᥠ´ +âĨ Ľ +âĨ ¶ +âĩ ¤ +âķ Ł +âĺ · +âļ IJ +ðŁ§ ´ +á¹ ³ +âĶ į +âĶ Ĵ +âĶ © +âĶ ¦ +â¾ µ +ઠľ +ઠ¤ +âĩ Ļ +âĶ ± +âķ Ģ +â½ Ĭ +ï½ Ł +ଠ¡ +ðł ® +ðł® · +âķ ĥ +â° Ķ +ãĬ ¦ +ðŁİ IJ +ãĩ ° +â¼ Ŀ +â¾ Ķ +â½ Ĵ +âł Ĵ +ï¨ ¦ +ï© Ĵ +ï¨ ² +ï© ĸ +ðĵı ¸ +ãĮ ĥ +ðĸ ¤ +ðĸ¤ IJ +ï¦ Ń +âĬ ħ +â¾ ³ +ä´ ¥ +ï© ķ +ðŁĮ Ķ +áŀ ĭ +âļ į +â¼ ĭ +ãİ ĺ +ðIJĮ ² +É © +áİ ij +âĨ ® +âĩ ĥ +âļ İ +ãĩ ± +ãĭ © +ãĮ ¶ +êĻ ª +ëİ ¬ +ï¨ IJ +ï¨ Ľ +ï© Ĭ +ï© į +ðĵ ħ +ðĵħ º +Ï ¡ +È ij +É Ĥ +Ô ĵ +ß İ +à´ § +áĢ ī +áĢ ĭ +áĢ ij +áĢ ł +áļ Ļ +ᨠĦ +ᨠ© +ᨠ¹ +á© ĵ +ᬠľ +á´ Ļ +áµ ij +âĤ Ń +âĨ ° +âľ ģ +â½ IJ +ãĭ ¯ +ãĮ ½ +íĨ ¢ +ï¤ ¿ +ðŁ Ĥ +ðŁĤ » +È Ĵ +Í º +Ô ¥ +Õ ij +Ú ¶ +à§ İ +ච® +ຠĸ +ຠľ +ຠ½ +áĥ » +áħ ¯ +áĭ ŀ +áĸ ķ +á ´Ī +ᶠĨ +Ḡľ +á¹ ¼ +á¿ ¨ +âĦ ĭ +âĦ Ń +âĪ ± +âĮ ĵ +âĶ ĩ +âĶ ¢ +â± ® +â² Ħ +ãĩ ¾ +ãĪ ¬ +ë¸ ¡ +ìIJ ī +íĻ Ľ +ðĿķ ª +Æ ¹ +Í ² +Ó ģ +Û ¼ +ঠ« +áħ Ł +áī Ĩ +áį Ī +Ạĸ +á½ ī +âĶ ¸ +â½ © +ê ľ +êľ ¥ +êµ ħ +ëĤ Ķ +ëĦ ł +ëĩ Ĺ +ëĻ Ŀ +ìļ ¯ +ìļ · +ìŁ Ľ +ì· IJ +íŁ ¬ +íŁ ® +íŁ ° +ï¦ Ĩ +ï¦ ± +ï² ŀ +ï³ ¤ +ï³ ¥ +ðIJĮ ¸ +ðĿĶ ı +ðĿķ ® +ðĿĺ £ +ঠĪ +âı ı +ãĦ ĸ +ê² ĩ +ëĸ ĺ +ëľ · +ëŀ Ĵ +ë¡ ĵ +ë¢ ī +ë£ ĥ +ë§ ĭ +ë² ĭ +ìĤ · +ìĪ ķ +ì Į¨ +ìĵ » +ìĸ Ĭ +ìĻ ¬ +ìĿ » +ì¦ ģ +ìµ ¤ +ì· ĥ +íĢ ľ +íħ ī +íį ł +íı ħ +íij ± +íķ ķ +íĸ ł +íĿ ķ +Æ Ļ +Æ ļ +Æ ŀ +Ç ĥ +Ç Ĭ +Ç ľ +Ç ¤ +Ç Ń +Ç ¹ +È Ģ +È ģ +È ħ +È ī +È Ĺ +È Ł +È ¤ +È ¥ +È ¨ +È µ +È º +È » +É Į +É ® +Ê ħ +Ê ¥ +Ê ¨ +Ë ĵ +Ë Ķ +Ë ł +Ë £ +Ë ¸ +Í ´ +Ï Ĺ +Ï ĺ +Ï Ļ +Ï ļ +Ï Ŀ +Ï ¨ +Ï ¬ +Ï ¾ +Ï ¿ +Ñ ª +Ò Ģ +Ò ľ +Ò ¼ +Ò ½ +Ó Ĥ +Ó ħ +Ó ĩ +Ó į +Ó ĸ +Ó Ł +Ó « +Ó ± +Ô Ĩ +Ô ĩ +Ô º +Õ ĭ +Ö ī +Ø Ī +Ø Ĭ +Ø ½ +Ø ¾ +Ù · +Ú Ĥ +Ú Ĭ +Ú ĸ +Ú Ĺ +Ú £ +Ú « +Ú ¸ +Û Ģ +Û į +Û ½ +Ü ī +Ü ¤ +Ý § +Ý ´ +Þ ĥ +Þ ¤ +Þ ¥ +ß ļ +ß Ľ +ß ¤ +àł į +àł ĵ +àł ³ +à¡ ¢ +ॠł +ৠł +ৠº +ਠĬ +ਠIJ +ਠ® +ਠ¯ +ਠ° +ਠ¸ +ઠĨ +ઠ³ +ઠµ +ઠ½ +ଠĮ +ଠĺ +ଠ½ +à® ĥ +à® ¸ +à° Ĩ +à° ķ +à° ¦ +ಠĨ +ಠĬ +ಠĮ +ಠIJ +ಠĽ +ಠ¤ +ಠ¦ +ಠª +ಠ² +ಠ¹ +à´ Ĩ +à´ ı +à´ Ĺ +à´ « +à´ ¹ +ൠº +ൠ½ +චħ +චĬ +චĶ +ච§ +ච« +ච° +༠Ħ +༠ħ +༠Ĭ +འĻ +འ¡ +འ§ +à¿ Ģ +à¿ Ļ +áĢ Ŀ +áĢ § +áĢ © +áĢ ¿ +áģ µ +áĤ ģ +áĤ ½ +áĥ Ĥ +áĥ ª +áĦ Ĭ +áĦ ¢ +áħ ¦ +áħ Ń +áĨ ® +áĨ ± +áĨ » +á ĩ +áĩ Ĥ +áĪ ħ +áĪ ī +áĪ Į +áĪ IJ +áĪ Ĵ +áĪ Ļ +áĪ ļ +áĪ ľ +áĪ ŀ +áĪ © +áĪ ³ +áĪ º +áĪ ½ +áī ħ +áī ¢ +áī ± +áī ´ +áĬ ĥ +áĬ į +áĬ ĸ +áĬ ® +áĬ ¸ +áĭ Ľ +áĭ Ŀ +áĭ ³ +áĮ ģ +áĮ ħ +áĮ ¥ +áĮ ¦ +á Į¨ +áį Ĭ +áį į +áį ķ +áį ĸ +áį ¢ +áį ¤ +áİ Ĵ +áİ ª +áı ģ +áı IJ +áı Ł +áIJ Ĥ +áIJ ĸ +áIJ Ŀ +áIJ ŀ +áIJ Ł +áIJ ł +áij ĸ +áĴ ĭ +áĴ į +áĴ ¡ +áĵ « +áĶ ķ +áķ ĭ +áķ ij +áķ Ļ +áķ ļ +áķ Ľ +áķ ¤ +áķ ¦ +áķ ® +áķ ¼ +áĸ ĵ +áĹ Ĺ +áĹ ¢ +áĹ ¯ +áĹ · +áĺ Ħ +áĺ ij +ἠĤ +ἠĻ +áŀ į +áł Ĩ +áł ¡ +áł ¦ +áł ® +áł ¯ +áł ² +áł · +á¡ į +á¡ ŀ +á¡ ¤ +á ¡´ +á¡ µ +ᤠĵ +ᥠĸ +ᥠ° +ᨠ¦ +ᨠ§ +ᨠ¨ +ᨠª +ᨠ¬ +ᨠ¯ +ᨠ³ +ᨠµ +á© ĥ +ᬠķ +áŃ £ +á ± +á± ļ +á² ł +á´ ĵ +á´ ¶ +áµ Ĥ +áµ Į +áµ ¥ +áµ ´ +ᶠĩ +ḠĪ +Ḡł +Ḡ§ +Ḡ´ +Ḡ¾ +á¹ Ģ +á¹ ĸ +á¹ Ł +á¹ ł +á¹ « +á¹ ± +á¹ · +á¹ ¿ +ẠĦ +Ạį +Ạij +áº Ĺ +á¼ ī +á¼ ĵ +á¼ Ń +á½ ĭ +á½ Ĵ +á½ ł +á½ £ +á¾ Ħ +á¾ ı +á¾ ij +á¾ Ĺ +á¾ ¦ +á¾ § +á¾ ¾ +á¿ Ħ +á¿ ĵ +á¿ ¡ +á¿ ¬ +âģ ļ +âĤ Į +âĦ ģ +âĦ Ķ +âĦ £ +âĦ § +âĦ ¯ +âĦ ° +âĦ ´ +âħ ħ +âĨ ľ +âĨ « +âĨ Ń +âĨ ± +âĨ ¹ +âĨ ½ +âĩ ĩ +âĩ ľ +âĩ µ +âĪ ī +âĪ Ĭ +âĪ ĸ +âĪ ľ +âĪ ¾ +âī Ģ +âī ĭ +âī Į +âī ĵ +âī ľ +âī ´ +âī ¿ +âĬ Ĭ +âĬ ĭ +âĬ Ķ +âĬ ĸ +âĬ £ +âĬ ¦ +âĭ İ +âĭ ª +âĭ ² +âĮ ¦ +âĮ § +âį º +âİ Ī +âİ ¨ +âİ ¬ +âİ ³ +âİ ¼ +âİ ¾ +âı Į +âı ļ +âı « +âı ¯ +âı µ +âĴ ľ +âĴ Ŀ +âĴ « +âĵ Ħ +âĵ Ĭ +âĵ Ļ +âĵ © +âĶ ij +âĶ Ļ +âĶ ļ +âĶ ¥ +âķ ħ +âķ ī +âķ į +âķ ı +âķ ŀ +âĸ ļ +âĸ ¯ +âĹ ĥ +âĹ ļ +âĹ ¬ +âĹ ´ +âĺ Ī +âĺ ¤ +âĺ ¥ +âĺ § +âĺ ¬ +âĻ ģ +âĻ ± +âļ ĥ +âļ Ħ +âļ ħ +âļ ı +âļ ļ +âļ ŀ +âļ Ł +âļ ± +âļ ² +âľ Ģ +âľ Ł +âľ ¢ +âĿ µ +⣠¡ +⣠¦ +⣠§ +⣠³ +⣠¾ +⣠¿ +âł ĩ +⤠Ħ +⤠º +⥠Ĥ +⥠¹ +⧠ī +⧠¼ +⧠½ +⨠į +⬠Ĭ +⬠Ł +âŃ ŀ +â® ŀ +â® ³ +⯠Ī +⯠ij +â± ł +â± ± +â² Ń +â´ ¹ +âµ ķ +⸠¾ +â º« +â¼ Ĩ +â¼ ł +â½ Ł +â½ ¼ +â¾ Ľ +â¾ § +â¿ ĥ +â¿ » +ãĤ ķ +ãĤ Ł +ãĦ Ľ +ãĦ ¡ +ãĦ ¶ +ãĦ º +ãħ Ĵ +ãħ Ł +ãĨ Ģ +ãĩ » +ãĪ ij +ãĪ Ń +ãĪ ® +ãĪ ³ +ãĪ ¹ +ãī ¥ +ãī ¦ +ãī ¹ +ãī ¿ +ãĬ ŀ +ãĬ ¨ +ãĭ ij +ãĭ ¥ +ãĭ ´ +ãĭ º +ãİ Ħ +ãİ ķ +ãİ ¯ +ãı Ĥ +ãı Ī +ãı ĵ +ãı ĸ +ãı ± +ãIJ ± +ãŁ ģ +ã ¢ +㢠¨ +ã ¨ +㨠³ +ã« ª +ã« ´ +㶠³ +㺠¾ +ä Ģ +äĢ Ģ +ä ĭ +äĭ Į +ä ĮĢ +äIJ Ģ +ä łĢ +ä ł +äł ¼ +ä § +ä§ ŀ +ä¨ ° +ä¨ º +ä ´Ģ +ä · +ä· ħ +ä ·¸ +ê Ĥ +êĤ « +ê Į +êĮ ¼ +ê į +êį ² +êĴ µ +ê ĵ +êĵ ½ +êĻ Ń +êĿ Ľ +êĿ ¥ +ê ŀ +êŀ Ĭ +ê¦ Ĩ +ê¦ ĩ +ê¦ Ł +ê¦ ¨ +ê§ Ī +ê © +ê© Ł +êª ĭ +êª ij +êª ķ +êª Ĺ +êª ľ +êª ® +êª ± +êª » +êª ¼ +ê« Ģ +ê« Ŀ +ê° ĥ +ê° ĺ +ê± ľ +ê² ĵ +ê² ļ +ê³ Ļ +ê³ ¾ +ê´ Ĺ +ê´ Ļ +êµ Ľ +ê¶ ĥ +ê¶ ķ +ê¶ ¨ +ê¸ © +ê¸ ¿ +ê ¹Ħ +ê¹ Ĩ +ê¹ ī +ê¹ ĵ +ê¹ ¢ +ê¹ £ +ê¹ ¸ +êº ³ +ê¿ ı +ê¿ ķ +ê¿ § +ëĢ © +ëģ ħ +ëĥ µ +ëĦ ĸ +ëĦ Ĺ +ëĦ ¢ +ëħ Ĥ +ëĨ IJ +ëĩ ľ +ëĪ ĭ +ëĪ ļ +ëī į +ëī ¨ +ëĬ ļ +ëĬ ¡ +ëĭ ľ +ëĭ ª +ëĮ ĺ +ëĮ ¤ +ëĮ ¸ +ëİ Ł +ëı ¨ +ëIJ Ħ +ëIJ ı +ëIJ ´ +ëIJ ¸ +ëij ģ +ëij ¿ +ëĴ ¨ +ëĵ · +ëĶ ® +ëĶ ² +ëķ § +ëĸ Ķ +ëĸ ª +ëĺ Ń +ëļ Ģ +ëļ ł +ëĽ Ķ +ëĽ © +ëľ ħ +ëŀ ķ +ëŀ ° +ëŁ IJ +ëł ¡ +ë¡ ŀ +ë¡ £ +ë¡ µ +ë£ Ħ +ë£ į +ë¤ ³ +ë¦ į +ë¦ ı +ë¦ ³ +ë§ Ħ +ë§ Ĩ +ë§ į +ë§ ľ +ë§ « +ë§ » +ë¨ ® +ë© Ĥ +ë© Ń +ëª ´ +ë¬ ľ +ë¬ ł +ë¬ « +ë¬ ¾ +ëŃ ¬ +ë® ĺ +ë® ¹ +ë¯ ķ +ë¯ ľ +ë° ¨ +ë° ª +ë± Ķ +ë² ĺ +ë² Ľ +ë² ± +ë² ´ +ë´ ½ +ëµ ¤ +ëµ ¨ +ë· Ĺ +ë· ĺ +ë¸ ĵ +ë¸ ľ +ë¹ ª +ëº ĥ +ëº ĺ +ëº µ +ë» ´ +ë¼ IJ +ë¾ Ķ +ìģ Ń +ìĤ ł +ìĤ ® +ìĥ ı +ìĥ Ļ +ìĦ º +ìħ ¢ +ìĨ Ģ +ìĨ ħ +ìĨ ¤ +ìĨ ¦ +ìĨ ¬ +ìĩ ± +ìĪ µ +ìĭ ¨ +ìĭ ´ +ìĮ ° +ìį ľ +ìİ Ĺ +ìİ ĺ +ìİ ¼ +ìij ī +ìij Ŀ +ìij » +ìĴ Ķ +ìĴ ¯ +ìĵ © +ìķ IJ +ìķ ĸ +ìĸ ł +ìĸ ¾ +ìĹ ĥ +ìĹ Ĺ +ìĹ ľ +ìĹ ¨ +ìĺ Ĥ +ìĺ Ħ +ìĺ ı +ìĺ ¾ +ìĺ ¿ +ìľ § +ìĿ IJ +ìĿ ĸ +ìĿ · +ìŀ į +ìŀ ı +ìŀ ¨ +ìŀ ª +ìŀ ³ +ìł ¡ +ìł ´ +ìł ¹ +ì¡ Ģ +ì¡ ª +ì¡ µ +ì¢ IJ +ì¢ ¨ +ì£ Į +ì£ Ļ +ì£ ³ +ì¦ ij +ì§ ¥ +ì§ ´ +ì§ ¾ +ì¨ ĵ +ì¨ ķ +ì© ° +ì© » +ì© ¼ +ìª Ĺ +ì¬ Ķ +ì¬ ĺ +ì® ® +ì¯ ķ +ì¯ ĺ +ì° İ +ì° ¯ +ì± ĥ +ì± µ +ì² § +ì² ® +ì² ¯ +ì³ ¬ +ì´ ĭ +ì´ ¢ +ìµ ¥ +ì¶ £ +ì¸ Ī +ì¸ Ļ +ìº ¤ +ìº Ń +ì» ½ +ì¼ Ļ +ì½ ¬ +ì¾ Ģ +ì¿ ħ +ì¿ ½ +íĢ ħ +íģ ¦ +íĤ ħ +íĥ ¶ +íĥ ¹ +íĦ Ķ +íħ £ +íĨ Ħ +íĨ § +íĨ ¹ +íĩ ¼ +íī ¤ +íĬ ½ +íĭ Ĥ +íĭ ij +íį Ī +íį Ļ +íį ¿ +íİ ¶ +íIJ Ŀ +íĴ ľ +íĵ Ŀ +íĵ ª +íĵ ± +íĵ · +íĵ ¼ +íĶ Ļ +íĶ ł +íķ ļ +íķ Ľ +íķ ŀ +íķ Ł +íķ § +íķ ¶ +íĸ Ĭ +íĸ ĭ +íĸ į +íĸ Ķ +íĸ ĺ +íĸ ¡ +íĸ ¬ +íĹ £ +íĹ ¿ +íĺ ĸ +íĺ Ń +íļ ° +íĽ į +íĽ ½ +íĿ Ł +íĿ Ń +íĿ ´ +íŀ ľ +ï¤ ī +ï¤ Ń +ï¤ ² +ï¤ µ +ï¤ ¼ +ï¥ Ģ +ï¥ ij +ï¥ Ĵ +ï¥ ķ +ï¥ ĺ +ï¥ Ļ +ï¥ « +ï¥ ¬ +ï¥ ° +ï ¥¿ +ï¦ ĭ +ï¦ ı +ï¦ Ķ +ï¦ ĸ +ï¦ ĺ +ï¦ Ľ +ï¦ ł +ï¦ ® +ï¦ ¯ +ï¦ º +ï¦ » +ï¦ ¾ +ï§ Ĩ +ï§ ĸ +ï§ Ľ +ï§ ŀ +ï§ Ł +ï§ § +ï§ ³ +ï§ º +ï§ ½ +ï¨ ĥ +ï¨ ļ +ï¨ ¢ +ï© Ł +ï¬ ¤ +ï¬ ¬ +ï¬ ¼ +ïŃ Ĵ +ïŃ ķ +ïŃ Ľ +ïŃ Ŀ +ïŃ ŀ +ïŃ Ł +ïŃ ¤ +ïŃ § +ïŃ ¨ +ïŃ ® +ïŃ ° +ïŃ ± +ïŃ · +ïŃ ¹ +ïŃ » +ï® Ģ +ï® ĥ +ï® Ħ +ï® ħ +ï® į +ï® Ĵ +ï® ĵ +ï® ķ +ï® ¦ +ï® ® +ï® ° +ï¯ ĵ +ï¯ ľ +ï¯ © +ï¯ ª +ï¯ ¬ +ï¯ Ń +ï¯ ® +ï¯ · +ï¯ ¹ +ï¯ » +ï¯ ¼ +ï° ĥ +ï° Į +ï° IJ +ï° ĺ +ï° Ļ +ï° ľ +ï° ŀ +ï° ¢ +ï° ® +ï° ° +ï° ¼ +ï° ¿ +ï± Ģ +ï± ģ +ï± Ī +ï± ĭ +ï± ı +ï± Ń +ï² Ģ +ï² ĩ +ï² Ī +ï² ĭ +ï² İ +ï² Ĵ +ï² ľ +ï² ł +ï² ¬ +ï² » +ï³ ĩ +ï³ Ķ +ï³ £ +ï³ « +ï´ ĺ +ï´ ° +ï´ ½ +ï ¶ +ï¶ ° +ï¸ ĸ +ï¸ ´ +ï¸ ¹ +ï¹ į +ï¹ Ĺ +ï¹ ¢ +ï¹ ¤ +ï¹ © +ï¹ ± +ï¾ ° +ï¿ Ĥ +ï¿ ® +ðIJĮ ° +ðIJĮ ¹ +ðIJĮ º +ðIJĮ ½ +ðIJį Ĥ +ðIJį ĥ +ðIJį Ħ +ðIJ İ +ðIJİ ¹ +ðIJ¤ Ĥ +ðIJ¤ į +ðIJ¤ ı +ðIJ¤ ĵ +ðIJŃ ī +ðIJŃ į +ðIJ° ĩ +ðIJ° ° +ðij Ĥ +ðijĤ Ħ +ðij ĺ +ðijĺ ģ +ðĴ Ģ +ðĴĢ ¸ +ðĴ ģ +ðĴģ º +ðĴ Ħ +ðĴĦ · +ðĴ Ĭ +ðĴĬ ij +ðĴ ĭ +ðĴĭ Ĺ +ð ĴĮ +ðĴĮ ¨ +ðĵĥ ¢ +ðĵĥ ° +ðĸ ł +ðĸł ļ +ðĿĦ ĥ +ðĿĦ ħ +ðĿĦ ķ +ðĿĦ Ļ +ðĿĦ ± +ðĿĦ ´ +ðĿĦ ¹ +ðĿħ İ +ðĿħ ª +ðĿĨ £ +ðĿĨ ³ +ðĿĨ ¹ +ðĿĩ Ĭ +ðĿĩ Ĺ +ðĿĩ ļ +ðĿĩ ľ +ðĿĩ ł +ðĿIJ ī +ðĿIJ ĸ +ðĿIJ ĺ +ðĿIJ £ +ðĿIJ ± +ðĿij Ĭ +ðĿij Ń +ðĿij ¼ +ðĿij ½ +ðĿĴ ° +ðĿĴ · +ðĿĴ ¿ +ðĿĵ ģ +ðĿĵ ĭ +ðĿĵ İ +ðĿĵ Ĵ +ðĿ ĵĺ +ðĿĵ ¢ +ðĿĵ ¦ +ðĿĵ « +ðĿĵ ¿ +ðĿĶ İ +ðĿĶ ± +ðĿĶ ´ +ðĿĶ · +ðĿĶ ¸ +ðĿĶ ½ +ðĿķ Ĥ +ðĿķ ĥ +ðĿķ ĭ +ðĿķ ı +ðĿķ IJ +ðĿķ ¥ +ðĿķ ´ +ðĿķ º +ðĿĸ IJ +ðĿĸ Ľ +ðĿĸ Ŀ +ðĿĸ ŀ +ðĿĹ © +ðĿĹ ³ +ðĿĹ ½ +ðĿĺ Ĭ +ðĿĺ ĭ +ðĿĺ Ķ +ðĿĺ ± +ðĿĺ ´ +ðĿĺ ¿ +ðĿĻ Ĵ +ðĿĻ Ŀ +ðĿĻ Ł +ðĿĻ ¬ +ðĿĻ Ń +ðĿĻ » +ðĿĻ ¾ +ðĿļ Ī +ðĿļ ĭ +ðĿļ ij +ðĿļ Ł +ðĿļ ł +ðĿļ £ +ðĿĽ ½ +ðĿľ Ĥ +ðĿľ Ķ +ðĿľ Ļ +ðŁ Ģ +ðŁĢ Ħ +ðŁĦ ² +ðŁĦ ¶ +ðŁħ IJ +ðŁħ ĸ +ðŁħ ļ +ðŁħ Ľ +ðŁħ ¦ +ðŁħ ¶ +ðŁħ » +ðŁħ ¼ +ðŁĨ ĥ +ðŁĨ Ĩ +ðŁĨ İ +ðŁĪ ¯ +ðŁĪ ² +ðŁĪ ¹ +ðŁĮ ĩ +ðŁĮ ĵ +ðŁį ĺ +ðŁİ ij +ðŁİ ¿ +ðŁı ı +ðŁı Ĵ +ðŁı © +ðŁı ¯ +ðŁIJ Ģ +ðŁij Ŀ +ðŁĴ ¹ +ðŁĴ º +ðŁĵ Ł +ðŁĵ ª +ðŁĵ ¼ +ðŁĶ Ģ +ðŁĶ Ĥ +ðŁĶ ĥ +ðŁĶ ĩ +ðŁĶ ĵ +ðŁĶ ¢ +ðŁĶ ¤ +ðŁĶ © +ðŁķ ĸ +ðŁķ ļ +ðŁķ ľ +ðŁķ Ŀ +ðŁķ ŀ +ðŁķ ł +ðŁķ ¢ +ðŁķ ³ +ðŁĸ ĩ +ðŁĸ ij +ðŁĸ ¶ +ðŁĹ ģ +Ñ ¨ +Ú İ +á¡ Į +Ḡ° +ẠĢ +á¼ ® +á½ Ŀ +âĦ ¬ +âļ § +⼠¤ +ã³ ¬ +êĻ ĭ +ê¸ ij +ëĶ ī +ëĹ į +ë¡ ij +ë¯ ij +ë» ħ +ë¼ Ŀ +ìĦ IJ +ìī ¡ +ìĭ ² +ìı ± +ìĹ ¤ +ìĿ © +ìĿ ¿ +ìŁ Ļ +ìł ° +ì¥ ī +íĬ Ń +íķ ® +ï® ı +ðŁħ ± +ðŁĨ Ĵ +ðŁķ ĭ +É ĺ +Ê ĵ +Õ ĥ +à´ ´ +འħ +áĨ º +áĪ Ĭ +áĪ ¨ +áĪ ¾ +áī IJ +áĮ ĥ +áĮ ½ +áĶ Ń +áł Ĥ +áł ¬ +ᨠ¸ +á© ĭ +ᶠı +á¾ Ķ +á¿ IJ +á¿ ļ +âĻ Ļ +âļ Ĥ +âļ Ĺ +â¡ ¢ +⤠¦ +ëĸ ° +ë¤ Ĥ +ë§ ł +ë± ĭ +ë± IJ +ìĽ ¢ +ìľ ¾ +ì³ ħ +ì» ģ +íģ » +íĥ Ļ +íĵ ĸ +íĵ Ń +íķ ± +íĽ ľ +ï¤ ħ +ï¤ Ĩ +ï¦ ĥ +ï§ © +ï¨ Ĥ +ðIJ¤ Ķ +ðIJŃ ĵ +ðIJ° ¼ +ðĿĵ ŀ +ðĿĵ ° +ðĿĻ ľ +ðĿļ ģ +ðŁħ ¢ +ðŁı ĩ +È ² +Ê ¶ +Ô Ī +Ô ij +Ý ĵ +Ý ¥ +ठij +ॠ± +ଠī +à° ³ +à° µ +ಠŁ +áĢ ı +áģ ¼ +áī ¨ +áĬ Ĵ +áĭ © +áĮ Ħ +áĮ Ķ +áIJ § +á ĴĮ +áĶ ħ +áĶ Ĭ +áł Ħ +ᨠģ +Ḡĥ +Ḡ» +âĶ ŀ +âĺ µ +âļ £ +â² ¢ +ãĪ ª +ä¶ µ +ê² Ļ +ê² ´ +ê³ Ĥ +ë¡ ¼ +ìĨ Ĭ +ì¼ ĩ +íĭ į +íĵ ¬ +íĵ ® +íĵ ¶ +íĵ » +ï¤ ¦ +ï¥ ł +ï¥ ± +ïŃ ² +ðIJŃ Ĭ +ðIJ ±ħ +ðĸ ¥ +ðĸ¥ ¨ +ðĿij ³ +ðĿĵ ķ +ðĿĵ ¬ +ðĿĵ ¹ +ðĿĵ ¾ +ðĿĶ ĵ +ðĿķ į +ðĿķ ¡ +ðĿķ ± +ðĿĸ ĸ +ðĿĺ ı +ðĿĺ IJ +ðĿĺ ļ +ðĿĻ ® +ðĿĻ ° +ðĿĻ ¸ +ðĿĻ º +ðĿĻ ¼ +ðĿĻ ½ +ðĿĻ ¿ +ðĿļ Ħ +ðĿļ ı +ðŁħ ħ +ðŁħ ĵ +Æ Ī +àł Į +áĻ ³ +á ļĮ +ἠħ +ἠIJ +ᤠĬ +ḠĬ +âĶ ½ +âķ Ĭ +⼠ĩ +⼠ı +âĿ ª +âĿ « +⣠° +ãĦ į +ãĦ ĵ +ãĦ § +ãħ ĸ +ãī « +ê¦ Ķ +ï± Ĭ +ຠĤ +áħ £ +ᥠĶ +ᥠ¤ +âĨ ¤ +âĨ · +âĩ ŀ +âĸ ¤ +âŀ ¶ +ãĪ ¼ +ï¨ · +ðĵı § +âĶ ² +âĢ ´ +âĴ Ł +âĴ ¡ +â° Ĥ +â° į +â° İ +â° IJ +â° ij +â° Ł +â° ł +â° ¡ +â¼ Ń +ãĬ ¥ +âĴ ł +â½ º +ãĩ º +ãĩ ½ +ï¨ Ĭ +áķ · +âį ¨ +⺠Ł +â½ Ĺ diff --git a/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/model.safetensors b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e9165bbe8188fc8cbabf222a7bb4547137764ba --- /dev/null +++ b/qian_sg_backup/qwen3_hyp/qwen3_hyp_p686m_t1_1_suite-owt_scaling_v3_family-qwen3_geometry_id-hyp_variant-base_init_slope-std/attempt4_20260310_173435/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f79cc7c4f0df48e5117ef87c4bbd8ccb1da18ca81d890801fa2456d6f9dbe2 +size 448359936