Source code for gevent._config

# Copyright (c) 2018 gevent. See LICENSE for details.
"""
gevent tunables.

This should be used as ``from gevent import config``. That variable
is an object of :class:`Config`.

.. versionadded:: 1.3a2

.. versionchanged:: 22.08.0
   Invoking this module like ``python -m gevent._config`` will
   print a help message about available configuration properties.
   This is handy to quickly look for environment variables.
"""

from __future__ import print_function, absolute_import, division

import importlib
import os
import textwrap

from gevent._compat import string_types
from gevent._compat import WIN

__all__ = [
    'config',
]

ALL_SETTINGS = []

class SettingType(type):
    # pylint:disable=bad-mcs-classmethod-argument

    def __new__(cls, name, bases, cls_dict):
        if name == 'Setting':
            return type.__new__(cls, name, bases, cls_dict)

        cls_dict["order"] = len(ALL_SETTINGS)
        if 'name' not in cls_dict:
            cls_dict['name'] = name.lower()

        if 'environment_key' not in cls_dict:
            cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper()


        new_class = type.__new__(cls, name, bases, cls_dict)
        new_class.fmt_desc(cls_dict.get("desc", ""))
        new_class.__doc__ = new_class.desc
        ALL_SETTINGS.append(new_class)

        if new_class.document:
            setting_name = cls_dict['name']

            def getter(self):
                return self.settings[setting_name].get()

            def setter(self, value): # pragma: no cover
                # The setter should never be hit, Config has a
                # __setattr__ that would override. But for the sake
                # of consistency we provide one.
                self.settings[setting_name].set(value)

            prop = property(getter, setter, doc=new_class.__doc__)

            setattr(Config, cls_dict['name'], prop)
        return new_class

    def fmt_desc(cls, desc):
        desc = textwrap.dedent(desc).strip()
        if hasattr(cls, 'shortname_map'):
            desc += (
                "\n\nThis is an importable value. It can be "
                "given as a string naming an importable object, "
                "or a list of strings in preference order and the first "
                "successfully importable object will be used. (Separate values "
                "in the environment variable with commas.) "
                "It can also be given as the callable object itself (in code). "
            )
            if cls.shortname_map:
                desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),)
        if getattr(cls.validate, '__doc__'):
            desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip()
        if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'):
            default = "`%s`" % (cls.default,)
        else:
            default = "`%r`" % (cls.default,)
        desc += "\n\nThe default value is %s" % (default,)
        desc += ("\n\nThe environment variable ``%s`` "
                 "can be used to control this." % (cls.environment_key,))
        setattr(cls, "desc", desc)
        return desc

def validate_invalid(value):
    raise ValueError("Not a valid value: %r" % (value,))

def validate_bool(value):
    """
    This is a boolean value.

    In the environment variable, it may be given as ``1``, ``true``,
    ``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or
    ``no`` for `False`.
    """
    if isinstance(value, string_types):
        value = value.lower().strip()
        if value in ('1', 'true', 'on', 'yes'):
            value = True
        elif value in ('0', 'false', 'off', 'no') or not value:
            value = False
        else:
            raise ValueError("Invalid boolean string: %r" % (value,))
    return bool(value)

def validate_anything(value):
    return value

convert_str_value_as_is = validate_anything

class Setting(object):
    name = None
    value = None
    validate = staticmethod(validate_invalid)
    default = None
    environment_key = None
    document = True

    desc = """\

    A long ReST description.

    The first line should be a single sentence.

    """

    def _convert(self, value):
        if isinstance(value, string_types):
            return value.split(',')
        return value

    def _default(self):
        result = os.environ.get(self.environment_key, self.default)
        result = self._convert(result)
        return result

    def get(self):
        # If we've been specifically set, return it
        if 'value' in self.__dict__:
            return self.value
        # Otherwise, read from the environment and reify
        # so we return consistent results.
        self.value = self.validate(self._default())
        return self.value

    def set(self, val):
        self.value = self.validate(self._convert(val))


Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__))

def make_settings():
    """
    Return fresh instances of all classes defined in `ALL_SETTINGS`.
    """
    settings = {}
    for setting_kind in ALL_SETTINGS:
        setting = setting_kind()
        assert setting.name not in settings
        settings[setting.name] = setting
    return settings


[docs]class Config(object): """ Global configuration for gevent. There is one instance of this object at ``gevent.config``. If you are going to make changes in code, instead of using the documented environment variables, you need to make the changes before using any parts of gevent that might need those settings. For example:: >>> from gevent import config >>> config.fileobject = 'thread' >>> from gevent import fileobject >>> fileobject.FileObject.__name__ 'FileObjectThread' .. versionadded:: 1.3a2 """ def __init__(self): self.settings = make_settings() def __getattr__(self, name): if name not in self.settings: raise AttributeError("No configuration setting for: %r" % name) return self.settings[name].get() def __setattr__(self, name, value): if name != "settings" and name in self.settings: self.set(name, value) else: super(Config, self).__setattr__(name, value) def set(self, name, value): if name not in self.settings: raise AttributeError("No configuration setting for: %r" % name) self.settings[name].set(value) def __dir__(self): return list(self.settings) def print_help(self): for k, v in self.settings.items(): print(k) print(textwrap.indent(v.__doc__.lstrip(), ' ' * 4)) print()
class ImportableSetting(object): def _import_one_of(self, candidates): assert isinstance(candidates, list) if not candidates: raise ImportError('Cannot import from empty list') for item in candidates[:-1]: try: return self._import_one(item) except ImportError: pass return self._import_one(candidates[-1]) def _import_one(self, path, _MISSING=object()): if not isinstance(path, string_types): return path if '.' not in path or '/' in path: raise ImportError("Cannot import %r. " "Required format: [package.]module.class. " "Or choose from %r" % (path, list(self.shortname_map))) module, item = path.rsplit('.', 1) module = importlib.import_module(module) x = getattr(module, item, _MISSING) if x is _MISSING: raise ImportError('Cannot import %r from %r' % (item, module)) return x shortname_map = {} def validate(self, value): if isinstance(value, type): return value return self._import_one_of([self.shortname_map.get(x, x) for x in value]) def get_options(self): result = {} for name, val in self.shortname_map.items(): try: result[name] = self._import_one(val) except ImportError as e: result[name] = e return result class BoolSettingMixin(object): validate = staticmethod(validate_bool) # Don't do string-to-list conversion. _convert = staticmethod(convert_str_value_as_is) class IntSettingMixin(object): # Don't do string-to-list conversion. def _convert(self, value): if value: return int(value) validate = staticmethod(validate_anything) class _PositiveValueMixin(object): def validate(self, value): if value is not None and value <= 0: raise ValueError("Must be positive") return value class FloatSettingMixin(_PositiveValueMixin): def _convert(self, value): if value: return float(value) class ByteCountSettingMixin(_PositiveValueMixin): _MULTIPLES = { # All keys must be the same size. 'kb': 1024, 'mb': 1024 * 1024, 'gb': 1024 * 1024 * 1024, } _SUFFIX_SIZE = 2 def _convert(self, value): if not value or not isinstance(value, str): return value value = value.lower() for s, m in self._MULTIPLES.items(): if value[-self._SUFFIX_SIZE:] == s: return int(value[:-self._SUFFIX_SIZE]) * m return int(value) class Resolver(ImportableSetting, Setting): desc = """\ The callable that will be used to create :attr:`gevent.hub.Hub.resolver`. See :doc:`dns` for more information. """ default = [ 'thread', 'dnspython', 'ares', 'block', ] shortname_map = { 'ares': 'gevent.resolver.ares.Resolver', 'thread': 'gevent.resolver.thread.Resolver', 'block': 'gevent.resolver.blocking.Resolver', 'dnspython': 'gevent.resolver.dnspython.Resolver', } class Threadpool(ImportableSetting, Setting): desc = """\ The kind of threadpool we use. """ default = 'gevent.threadpool.ThreadPool' class ThreadpoolIdleTaskTimeout(FloatSettingMixin, Setting): document = True name = 'threadpool_idle_task_timeout' environment_key = 'GEVENT_THREADPOOL_IDLE_TASK_TIMEOUT' desc = """\ How long threads in the default threadpool (used for DNS by default) are allowed to be idle before exiting. Use -1 for no timeout. .. versionadded:: 22.08.0 """ # This value is picked pretty much arbitrarily. # We want to balance performance (keeping threads around) # with memory/cpu usage (letting threads go). default = 5.0 class Loop(ImportableSetting, Setting): desc = """\ The kind of the loop we use. On Windows, this defaults to libuv, while on other platforms it defaults to libev. """ default = [ 'libev-cext', 'libev-cffi', 'libuv-cffi', ] if not WIN else [ 'libuv-cffi', 'libev-cext', 'libev-cffi', ] shortname_map = { 'libev-cext': 'gevent.libev.corecext.loop', 'libev-cffi': 'gevent.libev.corecffi.loop', 'libuv-cffi': 'gevent.libuv.loop.loop', } shortname_map['libuv'] = shortname_map['libuv-cffi'] class FormatContext(ImportableSetting, Setting): name = 'format_context' # using pprint.pformat can override custom __repr__ methods on dict/list # subclasses, which can be a security concern default = 'pprint.saferepr' class LibevBackend(Setting): name = 'libev_backend' environment_key = 'GEVENT_BACKEND' desc = """\ The backend for libev, such as 'select' """ default = None validate = staticmethod(validate_anything) class FileObject(ImportableSetting, Setting): desc = """\ The kind of ``FileObject`` we will use. See :mod:`gevent.fileobject` for a detailed description. """ environment_key = 'GEVENT_FILE' default = [ 'posix', 'thread', ] shortname_map = { 'thread': 'gevent._fileobjectcommon.FileObjectThread', 'posix': 'gevent._fileobjectposix.FileObjectPosix', 'block': 'gevent._fileobjectcommon.FileObjectBlock' } class WatchChildren(BoolSettingMixin, Setting): desc = """\ Should we *not* watch children with the event loop watchers? This is an advanced setting. See :mod:`gevent.os` for a detailed description. """ name = 'disable_watch_children' environment_key = 'GEVENT_NOWAITPID' default = False class TraceMalloc(IntSettingMixin, Setting): name = 'trace_malloc' environment_key = 'PYTHONTRACEMALLOC' default = False desc = """\ Should FFI objects track their allocation? This is only useful for low-level debugging. On Python 3, this environment variable is built in to the interpreter, and it may also be set with the ``-X tracemalloc`` command line argument. On Python 2, gevent interprets this argument and adds extra tracking information for FFI objects. """ class TrackGreenletTree(BoolSettingMixin, Setting): name = 'track_greenlet_tree' environment_key = 'GEVENT_TRACK_GREENLET_TREE' default = True desc = """\ Should `Greenlet` objects track their spawning tree? Setting this to a false value will make spawning `Greenlet` objects and using `spawn_raw` faster, but the ``spawning_greenlet``, ``spawn_tree_locals`` and ``spawning_stack`` will not be captured. Setting this to a false value can also reduce memory usage because capturing the stack captures some information about Python frames. .. versionadded:: 1.3b1 """ ## Monitoring settings # All env keys should begin with GEVENT_MONITOR class MonitorThread(BoolSettingMixin, Setting): name = 'monitor_thread' environment_key = 'GEVENT_MONITOR_THREAD_ENABLE' default = False desc = """\ Should each hub start a native OS thread to monitor for problems? Such a thread will periodically check to see if the event loop is blocked for longer than `max_blocking_time`, producing output on the hub's exception stream (stderr by default) if it detects this condition. If this setting is true, then this thread will be created the first time the hub is switched to, or you can call :meth:`gevent.hub.Hub.start_periodic_monitoring_thread` at any time to create it (from the same thread that will run the hub). That function will return an instance of :class:`gevent.events.IPeriodicMonitorThread` to which you can add your own monitoring functions. That function also emits an event of :class:`gevent.events.PeriodicMonitorThreadStartedEvent`. .. seealso:: `max_blocking_time` .. versionadded:: 1.3b1 """ class MaxBlockingTime(FloatSettingMixin, Setting): name = 'max_blocking_time' # This environment key doesn't follow the convention because it's # meant to match a key used by existing projects environment_key = 'GEVENT_MAX_BLOCKING_TIME' default = 0.1 desc = """\ If the `monitor_thread` is enabled, this is approximately how long (in seconds) the event loop will be allowed to block before a warning is issued. This function depends on using `greenlet.settrace`, so installing your own trace function after starting the monitoring thread will cause this feature to misbehave unless you call the function returned by `greenlet.settrace`. If you install a tracing function *before* the monitoring thread is started, it will still be called. .. note:: In the unlikely event of creating and using multiple different gevent hubs in the same native thread in a short period of time, especially without destroying the hubs, false positives may be reported. .. versionadded:: 1.3b1 """ class MonitorMemoryPeriod(FloatSettingMixin, Setting): name = 'memory_monitor_period' environment_key = 'GEVENT_MONITOR_MEMORY_PERIOD' default = 5 desc = """\ If `monitor_thread` is enabled, this is approximately how long (in seconds) we will go between checking the processes memory usage. Checking the memory usage is relatively expensive on some operating systems, so this should not be too low. gevent will place a floor value on it. """ class MonitorMemoryMaxUsage(ByteCountSettingMixin, Setting): name = 'max_memory_usage' environment_key = 'GEVENT_MONITOR_MEMORY_MAX' default = None desc = """\ If `monitor_thread` is enabled, then if memory usage exceeds this amount (in bytes), events will be emitted. See `gevent.events`. In the environment variable, you can use a suffix of 'kb', 'mb' or 'gb' to specify the value in kilobytes, megabytes or gigibytes. There is no default value for this setting. If you wish to cap memory usage, you must choose a value. """ # The ares settings are all interpreted by # gevent/resolver/ares.pyx, so we don't do # any validation here. class AresSettingMixin(object): document = False @property def kwarg_name(self): return self.name[5:] validate = staticmethod(validate_anything) _convert = staticmethod(convert_str_value_as_is) class AresFlags(AresSettingMixin, Setting): name = 'ares_flags' default = None environment_key = 'GEVENTARES_FLAGS' class AresTimeout(AresSettingMixin, Setting): document = True name = 'ares_timeout' default = None environment_key = 'GEVENTARES_TIMEOUT' desc = """\ .. deprecated:: 1.3a2 Prefer the :attr:`resolver_timeout` setting. If both are set, the results are not defined. """ class AresTries(AresSettingMixin, Setting): name = 'ares_tries' default = None environment_key = 'GEVENTARES_TRIES' class AresNdots(AresSettingMixin, Setting): name = 'ares_ndots' default = None environment_key = 'GEVENTARES_NDOTS' class AresUDPPort(AresSettingMixin, Setting): name = 'ares_udp_port' default = None environment_key = 'GEVENTARES_UDP_PORT' class AresTCPPort(AresSettingMixin, Setting): name = 'ares_tcp_port' default = None environment_key = 'GEVENTARES_TCP_PORT' class AresServers(AresSettingMixin, Setting): document = True name = 'ares_servers' default = None environment_key = 'GEVENTARES_SERVERS' desc = """\ A list of strings giving the IP addresses of nameservers for the ares resolver. In the environment variable, these strings are separated by commas. .. deprecated:: 1.3a2 Prefer the :attr:`resolver_nameservers` setting. If both are set, the results are not defined. """ # Generic nameservers, works for dnspython and ares. class ResolverNameservers(AresSettingMixin, Setting): document = True name = 'resolver_nameservers' default = None environment_key = 'GEVENT_RESOLVER_NAMESERVERS' desc = """\ A list of strings giving the IP addresses of nameservers for the (non-system) resolver. In the environment variable, these strings are separated by commas. .. rubric:: Resolver Behaviour * blocking Ignored * Threaded Ignored * dnspython If this setting is not given, the dnspython resolver will load nameservers to use from ``/etc/resolv.conf`` or the Windows registry. This setting replaces any nameservers read from those means. Note that the file and registry are still read for other settings. .. caution:: dnspython does not validate the members of the list. An improper address (such as a hostname instead of IP) has undefined results, including hanging the process. * ares Similar to dnspython, but with more platform and compile-time options. ares validates that the members of the list are valid addresses. """ # Normal string-to-list rules. But still validate_anything. _convert = Setting._convert # TODO: In the future, support reading a resolv.conf file # *other* than /etc/resolv.conf, and do that both on Windows # and other platforms. Also offer the option to disable the system # configuration entirely. @property def kwarg_name(self): return 'servers' # Generic timeout, works for dnspython and ares class ResolverTimeout(FloatSettingMixin, AresSettingMixin, Setting): document = True name = 'resolver_timeout' environment_key = 'GEVENT_RESOLVER_TIMEOUT' desc = """\ The total amount of time that the DNS resolver will spend making queries. Only the ares and dnspython resolvers support this. .. versionadded:: 1.3a2 """ @property def kwarg_name(self): return 'timeout' config = Config() # Go ahead and attempt to import the loop when this class is # instantiated. The hub won't work if the loop can't be found. This # can solve problems with the class being imported from multiple # threads at once, leading to one of the imports failing. # factories are themselves handled lazily. See #687. # Don't cache it though, in case the user re-configures through the # API. try: Loop().get() except ImportError: # pragma: no cover pass if __name__ == '__main__': config.print_help()