Module bigbench.api.util

Utility functions for BIG-bench.

# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utility functions for BIG-bench.
"""

import re
import abc
import importlib
import bigbench.api.task as task
import bigbench.api.json_task as json_task


def postprocess_output(text, max_length, stop_string, output_regex):
    """
    Modify model output to satisfy stop_string and output_regex keywords.

    Args:
      text: A string or list of strings containing model outputs.
      max_length: Model output will be truncated to be at most this length.
      stop_string: Model output will be truncated to the shortest string
        which includes stop_string. If None, no truncation will be performed.
        e.g. if stop_string='.', the model output "The apple is on the
        ground. What is" will be truncated to "The apple is on the ground."
        before being returned.
      output_regex: Rather than returning the full output, return the first
        match to the python regular expression output_regex in the output
        string. If there is no match, an empty string will be returned. If
        output_regex=None, no regular expression matching will be performed.
        e.g. if output_regex=r"\d+", and the model output is "= 42. 7*7 =
        49. 7*8 =", then "42" will be returned.

    Returns: A string or list of strings, all processed as described for each
      argument.
    """

    if isinstance(text, list):
        return [
            postprocess_output(mo, max_length, stop_string, output_regex) for mo in text
        ]

    if not isinstance(text, str):
        raise ValueError("model_output has unexpected type %s" % type(model_output))

    # truncate at max_length
    if max_length:
        text = text[:max_length]

    # Remove all text after any stop_string
    if stop_string:
        index = text.find(stop_string)
        if index > 0:
            text = text[: index + len(stop_string)]

    # extract substring matching regex (empty string for no match)
    if output_regex:
        text = next(iter(re.findall(output_regex, text)), "")

    return text


def load_json_task(json_path):
    this_task = json_task.JsonTask(json_path)
    return this_task


def load_programmatic_task(task_module_name: str) -> task.Task:
    """Gets the bigbench.api.Task instance from the given module.

    Args:
      task_module_name: the module name to search, like
                        bigbench.benchmark_tasks.simple_arithmetic

    Returns:
      Task instance
    """

    this_task_class = importlib.import_module(task_module_name)
    task_class_index = -1
    for i, k in enumerate(dir(this_task_class)):
        this_method = getattr(this_task_class, k)
        if isinstance(this_method, abc.ABCMeta):
            this_method_object = this_method()
            if issubclass(this_method_object.__class__, task.Task):
                task_class_index = i

    assert (
        task_class_index != -1
    ), f"Task class definition that extends bigbench.api. Task not found in module {task_module_name}"

    return getattr(this_task_class, dir(this_task_class)[task_class_index])()

Functions

def load_json_task(json_path)

Expand source code

def load_json_task(json_path):
    this_task = json_task.JsonTask(json_path)
    return this_task

def load_programmatic_task(task_module_name: str) ‑> Task

Gets the bigbench.api.Task instance from the given module.

Args

task_module_name

the module name to search, like bigbench.benchmark_tasks.simple_arithmetic

Returns

Task instance

Expand source code

def load_programmatic_task(task_module_name: str) -> task.Task:
    """Gets the bigbench.api.Task instance from the given module.

    Args:
      task_module_name: the module name to search, like
                        bigbench.benchmark_tasks.simple_arithmetic

    Returns:
      Task instance
    """

    this_task_class = importlib.import_module(task_module_name)
    task_class_index = -1
    for i, k in enumerate(dir(this_task_class)):
        this_method = getattr(this_task_class, k)
        if isinstance(this_method, abc.ABCMeta):
            this_method_object = this_method()
            if issubclass(this_method_object.__class__, task.Task):
                task_class_index = i

    assert (
        task_class_index != -1
    ), f"Task class definition that extends bigbench.api. Task not found in module {task_module_name}"

    return getattr(this_task_class, dir(this_task_class)[task_class_index])()

def postprocess_output(text, max_length, stop_string, output_regex)

Modify model output to satisfy stop_string and output_regex keywords.

Args

text

A string or list of strings containing model outputs.

max_length

Model output will be truncated to be at most this length.

stop_string

Model output will be truncated to the shortest string which includes stop_string. If None, no truncation will be performed. e.g. if stop_string='.', the model output "The apple is on the ground. What is" will be truncated to "The apple is on the ground." before being returned.

output_regex

Rather than returning the full output, return the first match to the python regular expression output_regex in the output string. If there is no match, an empty string will be returned. If output_regex=None, no regular expression matching will be performed. e.g. if output_regex=r"\d+", and the model output is "= 42. 77 = 49. 78 =", then "42" will be returned.

Returns: A string or list of strings, all processed as described for each argument.

Expand source code

def postprocess_output(text, max_length, stop_string, output_regex):
    """
    Modify model output to satisfy stop_string and output_regex keywords.

    Args:
      text: A string or list of strings containing model outputs.
      max_length: Model output will be truncated to be at most this length.
      stop_string: Model output will be truncated to the shortest string
        which includes stop_string. If None, no truncation will be performed.
        e.g. if stop_string='.', the model output "The apple is on the
        ground. What is" will be truncated to "The apple is on the ground."
        before being returned.
      output_regex: Rather than returning the full output, return the first
        match to the python regular expression output_regex in the output
        string. If there is no match, an empty string will be returned. If
        output_regex=None, no regular expression matching will be performed.
        e.g. if output_regex=r"\d+", and the model output is "= 42. 7*7 =
        49. 7*8 =", then "42" will be returned.

    Returns: A string or list of strings, all processed as described for each
      argument.
    """

    if isinstance(text, list):
        return [
            postprocess_output(mo, max_length, stop_string, output_regex) for mo in text
        ]

    if not isinstance(text, str):
        raise ValueError("model_output has unexpected type %s" % type(model_output))

    # truncate at max_length
    if max_length:
        text = text[:max_length]

    # Remove all text after any stop_string
    if stop_string:
        index = text.find(stop_string)
        if index > 0:
            text = text[: index + len(stop_string)]

    # extract substring matching regex (empty string for no match)
    if output_regex:
        text = next(iter(re.findall(output_regex, text)), "")

    return text