Creating command-line Tools

[1]:
from ctapipe.core import Tool, Component, TelescopeComponent
from ctapipe.core.traits import (
    Integer, Float, List, Dict, Unicode,
    TraitError, observe, FloatTelescopeParameter,
    Path,
)
import logging
from time import sleep
from astropy import units as u
from ctapipe.utils import get_dataset_path
[2]:
GAMMA_FILE = get_dataset_path('gamma_test.simtel.gz')

see https://github.com/ipython/traitlets/blob/master/examples/myapp.py

Setup:

Create a few Components that we will use later in a Tool:

[3]:
class MyComponent(Component):
    """ A Component that does stuff """

    value = Integer(default_value=-1, help="Value to use").tag(config=True)

    def do_thing(self):
        self.log.debug("Did thing")


# in order to have 2 of the same components at once
class SecondaryMyComponent(MyComponent):
    pass


class AdvancedComponent(Component):
    """ An advanced technique """

    value1 = Integer(default_value=-1, help="Value to use").tag(config=True)
    infile = Path(
        help="input file name",
        exists=None,  # set to True to require existing, False for requiring non-existing
        directory_ok=False,
    ).tag(config=True)
    outfile = Path(
        help="output file name",
        exists=False, directory_ok=False
    ).tag(config=True)

    def __init__(self, config=None, parent=None, **kwargs):
        super().__init__(config=config, parent=parent, **kwargs)
        # components can have sub components, but these must have
        # then parent=self as argument and be assigned as member
        # so the full config can be received later
        self.subcompent = MyComponent(parent=self)

    @observe("outfile")
    def on_outfile_changed(self, change):
        self.log.warning("Outfile was changed to '{}'".format(change))


class TelescopeWiseComponent(TelescopeComponent):
    """ a component that contains parameters that are per-telescope configurable """

    param = FloatTelescopeParameter(
        help="Something configurable with telescope patterns", default_value=5.0
    ).tag(config=True)
[4]:
MyComponent()
[4]:
MyComponent

A Component that does stuff

value -1 Value to use (default: -1)
[5]:
AdvancedComponent(infile="test.foo", outfile="out.foo")
Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/runner/work/ctapipe/ctapipe/docs/examples/out.foo'), 'owner': <__main__.AdvancedComponent object at 0x7fcc5c4c77c0>, 'type': 'change'}'
[5]:
AdvancedComponent

An advanced technique

infile /home/runner/work/ctapipe/ctapipe/docs/examples/test.foo input file name (default: traitlets.Undefined)
outfile /home/runner/work/ctapipe/ctapipe/docs/examples/out.foo output file name (default: traitlets.Undefined)
value1 -1 Value to use (default: -1)

TelescopeComponents need to have a subarray given to them in order to work (since they need one to turn a TelescopeParameter into a concrete list of values for each telescope. Here we will give a dummy one:

[6]:
from ctapipe.instrument import SubarrayDescription, TelescopeDescription

subarray = SubarrayDescription(
    "Junk",
    tel_positions={1: (0.0, 0.0, 0.0) * u.m, 2: (1.0, 1.0, 0.0) * u.m},
    tel_descriptions={
        1: TelescopeDescription.from_name("LST", "LSTCam"),
        2: TelescopeDescription.from_name("MST", "NectarCam"),
    },
)
subarray.info()
Subarray : Junk
Num Tels : 2
Footprint: 0.00 km2

       Type       Count Tel IDs
----------------- ----- -------
   LST_LST_LSTCam     1 1
MST_MST_NectarCam     1 2
[7]:
TelescopeWiseComponent(subarray=subarray)
[7]:
TelescopeWiseComponent

a component that contains parameters that are per-telescope configurable

param [('type', '*', 5.0)] Something configurable with telescope patterns (default: traitlets.Undefined)

This TelescopeParameters can then be set using a list of patterns like:

component.param = [
    ("type", "LST*",3.0),
    ("type", "MST*", 2.0),
    (id, 25, 4.0)
]

These get translated into per-telescope-id values once the subarray is registered. After that one acccess the per-telescope id values via:

component.param.tel[tel_id]

Now create an executable Tool that contains the Components

[8]:
class MyTool(Tool):
    name="mytool"
    description="do some things and stuff"
    aliases = dict(
        infile='AdvancedComponent.infile',
        outfile='AdvancedComponent.outfile',
        iterations='MyTool.iterations'
    )

    # Which classes are registered for configuration
    classes = [MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent]

    # local configuration parameters
    iterations = Integer(5,help="Number of times to run",allow_none=False).tag(config=True)

    def setup(self):
        self.comp = MyComponent(parent=self)
        self.comp2 = SecondaryMyComponent(parent=self)
        self.comp3 = TelescopeWiseComponent(parent=self, subarray=subarray)
        self.advanced = AdvancedComponent(parent=self)


    def start(self):
        self.log.info("Performing {} iterations...".format(self.iterations))
        for ii in range(self.iterations):
            self.log.info("ITERATION {}".format(ii))
            self.comp.do_thing()
            self.comp2.do_thing()
            sleep(0.1)

    def finish(self):
        self.log.warning("Shutting down.")

Get Help info

The following allows you to print the help info within a Jupyter notebook, but this same inforamtion would be displayed if the user types:

mytool --help
[9]:
tool = MyTool()
tool
[9]:
MyTool

do some things and stuff

config_files [] (default: traitlets.Undefined)
iterations 5 Number of times to run (default: 5)
log_config {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': , 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': , 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}} (default: traitlets.Undefined)
log_datefmt %Y-%m-%d %H:%M:%S The date format used by logging formatters for %(asctime)s (default: %Y-%m-%d %H:%M:%S)
log_file None Filename for the log (default: None)
log_file_level INFO Logging Level for File Logging (default: INFO)
log_format [%(name)s]%(highlevel)s %(message)s The Logging format template (default: [%(name)s]%(highlevel)s %(message)s)
log_level 30 Set the log level by value or name. (default: 30)
logging_config {} Configure additional log handlers. The default stderr logs handler is configured by the log_level, log_datefmt and log_format settings. This configuration can be used to configure additional handlers (e.g. to output the log to a file) or for finer control over the default handlers. If provided this should be a logging configuration dictionary, for more information see: https://docs.python.org/3/library/logging.config.html#logging-config-dictschema This dictionary is merged with the base logging configuration which defines the following: * A logging formatter intended for interactive use called ``console``. * A logging handler that writes to stderr called ``console`` which uses the formatter ``console``. * A logger with the name of this application set to ``DEBUG`` level. This example adds a new handler that writes to a file: .. code-block:: python c.Application.logging_configuration = { 'handlers': { 'file': { 'class': 'logging.FileHandler', 'level': 'DEBUG', 'filename': '', } }, 'loggers': { '': { 'level': 'DEBUG', # NOTE: if you don't list the default "console" # handler here then it will be disabled 'handlers': ['console', 'file'], }, } } (default: traitlets.Undefined)
provenance_log /home/runner/work/ctapipe/ctapipe/docs/examples/mytool.provenance.log (default: traitlets.Undefined)
quiet False (default: False)
show_config False Instead of starting the Application, dump configuration to stdout (default: False)
show_config_json False Instead of starting the Application, dump configuration to stdout (as JSON) (default: False)

Components: MyTool, MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent

[10]:
tool.print_help()
do some things and stuff

Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
    <cmd> --help-all

--debug
    Set log-level to debug, for the most verbose logging.
    Equivalent to: [--Application.log_level=10]
--show-config
    Show the application's configuration (human-readable format)
    Equivalent to: [--Application.show_config=True]
--show-config-json
    Show the application's configuration (json format)
    Equivalent to: [--Application.show_config_json=True]
-q, --quiet
    Disable console logging.
    Equivalent to: [--Tool.quiet=True]
-v, --verbose
    Set log level to DEBUG
    Equivalent to: [--Tool.log_level=DEBUG]
-c, --config=<list-item-1>...
    Default: []
    Equivalent to: [--Tool.config_files]
--log-level=<Enum>
    Set the log level by value or name.
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 30
    Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
    Filename for the log
    Default: None
    Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
    Logging Level for File Logging
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 'INFO'
    Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
    Default: traitlets.Undefined
    Equivalent to: [--Tool.provenance_log]
--infile=<Path>
    input file name
    Default: traitlets.Undefined
    Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
    output file name
    Default: traitlets.Undefined
    Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Int>
    Number of times to run
    Default: 5
    Equivalent to: [--MyTool.iterations]

To see all available configurables, use `--help-all`.

The following is equivalant to the user typing mytool --help-all

[11]:
tool.print_help(classes=True)
do some things and stuff

Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
    <cmd> --help-all

--debug
    Set log-level to debug, for the most verbose logging.
    Equivalent to: [--Application.log_level=10]
--show-config
    Show the application's configuration (human-readable format)
    Equivalent to: [--Application.show_config=True]
--show-config-json
    Show the application's configuration (json format)
    Equivalent to: [--Application.show_config_json=True]
-q, --quiet
    Disable console logging.
    Equivalent to: [--Tool.quiet=True]
-v, --verbose
    Set log level to DEBUG
    Equivalent to: [--Tool.log_level=DEBUG]
-c, --config=<list-item-1>...
    Default: []
    Equivalent to: [--Tool.config_files]
--log-level=<Enum>
    Set the log level by value or name.
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 30
    Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
    Filename for the log
    Default: None
    Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
    Logging Level for File Logging
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 'INFO'
    Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
    Default: traitlets.Undefined
    Equivalent to: [--Tool.provenance_log]
--infile=<Path>
    input file name
    Default: traitlets.Undefined
    Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
    output file name
    Default: traitlets.Undefined
    Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Int>
    Number of times to run
    Default: 5
    Equivalent to: [--MyTool.iterations]

Class options
=============
The command-line option below sets the respective configurable class-parameter:
    --Class.parameter=value
This line is evaluated in Python, so simple expressions are allowed.
For instance, to set `C.a=[0,1,2]`, you may type this:
    --C.a='range(3)'

Application(SingletonConfigurable) options
------------------------------------------
--Application.log_datefmt=<Unicode>
    The date format used by logging formatters for %(asctime)s
    Default: '%Y-%m-%d %H:%M:%S'
--Application.log_format=<Unicode>
    The Logging format template
    Default: '[%(name)s]%(highlevel)s %(message)s'
--Application.log_level=<Enum>
    Set the log level by value or name.
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 30
--Application.logging_config=<key-1>=<value-1>...
    Configure additional log handlers.
    The default stderr logs handler is configured by the log_level, log_datefmt
    and log_format settings.
    This configuration can be used to configure additional handlers (e.g. to
    output the log to a file) or for finer control over the default handlers.
    If provided this should be a logging configuration dictionary, for more
    information see:
    https://docs.python.org/3/library/logging.config.html#logging-config-
    dictschema
    This dictionary is merged with the base logging configuration which defines
    the following:
    * A logging formatter intended for interactive use called
      ``console``.
    * A logging handler that writes to stderr called
      ``console`` which uses the formatter ``console``.
    * A logger with the name of this application set to ``DEBUG``
      level.
    This example adds a new handler that writes to a file:
    .. code-block:: python
       c.Application.logging_configuration = {
           'handlers': {
               'file': {
                   'class': 'logging.FileHandler',
                   'level': 'DEBUG',
                   'filename': '<path/to/file>',
               }
           },
           'loggers': {
               '<application-name>': {
                   'level': 'DEBUG',
                   # NOTE: if you don't list the default "console"
                   # handler here then it will be disabled
                   'handlers': ['console', 'file'],
               },
           }
       }
    Default: {}
--Application.show_config=<Bool>
    Instead of starting the Application, dump configuration to stdout
    Default: False
--Application.show_config_json=<Bool>
    Instead of starting the Application, dump configuration to stdout (as JSON)
    Default: False

Tool(Application) options
-------------------------
--Tool.config_files=<list-item-1>...
    Default: []
--Tool.log_config=<key-1>=<value-1>...
    Default: {'version': 1, 'disable_existing_loggers': False, 'formatters...
--Tool.log_datefmt=<Unicode>
    The date format used by logging formatters for %(asctime)s
    Default: '%Y-%m-%d %H:%M:%S'
--Tool.log_file=<Path>
    Filename for the log
    Default: None
--Tool.log_file_level=<Enum>
    Logging Level for File Logging
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 'INFO'
--Tool.log_format=<Unicode>
    The Logging format template
    Default: '[%(name)s]%(highlevel)s %(message)s'
--Tool.log_level=<Enum>
    Set the log level by value or name.
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 30
--Tool.logging_config=<key-1>=<value-1>...
    Configure additional log handlers.
    The default stderr logs handler is configured by the log_level, log_datefmt
    and log_format settings.
    This configuration can be used to configure additional handlers (e.g. to
    output the log to a file) or for finer control over the default handlers.
    If provided this should be a logging configuration dictionary, for more
    information see:
    https://docs.python.org/3/library/logging.config.html#logging-config-
    dictschema
    This dictionary is merged with the base logging configuration which defines
    the following:
    * A logging formatter intended for interactive use called
      ``console``.
    * A logging handler that writes to stderr called
      ``console`` which uses the formatter ``console``.
    * A logger with the name of this application set to ``DEBUG``
      level.
    This example adds a new handler that writes to a file:
    .. code-block:: python
       c.Application.logging_configuration = {
           'handlers': {
               'file': {
                   'class': 'logging.FileHandler',
                   'level': 'DEBUG',
                   'filename': '<path/to/file>',
               }
           },
           'loggers': {
               '<application-name>': {
                   'level': 'DEBUG',
                   # NOTE: if you don't list the default "console"
                   # handler here then it will be disabled
                   'handlers': ['console', 'file'],
               },
           }
       }
    Default: {}
--Tool.provenance_log=<Path>
    Default: traitlets.Undefined
--Tool.quiet=<Bool>
    Default: False
--Tool.show_config=<Bool>
    Instead of starting the Application, dump configuration to stdout
    Default: False
--Tool.show_config_json=<Bool>
    Instead of starting the Application, dump configuration to stdout (as JSON)
    Default: False

MyTool(Tool) options
--------------------
--MyTool.config_files=<list-item-1>...
    Default: []
--MyTool.iterations=<Int>
    Number of times to run
    Default: 5
--MyTool.log_config=<key-1>=<value-1>...
    Default: {'version': 1, 'disable_existing_loggers': False, 'formatters...
--MyTool.log_datefmt=<Unicode>
    The date format used by logging formatters for %(asctime)s
    Default: '%Y-%m-%d %H:%M:%S'
--MyTool.log_file=<Path>
    Filename for the log
    Default: None
--MyTool.log_file_level=<Enum>
    Logging Level for File Logging
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 'INFO'
--MyTool.log_format=<Unicode>
    The Logging format template
    Default: '[%(name)s]%(highlevel)s %(message)s'
--MyTool.log_level=<Enum>
    Set the log level by value or name.
    Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
    Default: 30
--MyTool.logging_config=<key-1>=<value-1>...
    Configure additional log handlers.
    The default stderr logs handler is configured by the log_level, log_datefmt
    and log_format settings.
    This configuration can be used to configure additional handlers (e.g. to
    output the log to a file) or for finer control over the default handlers.
    If provided this should be a logging configuration dictionary, for more
    information see:
    https://docs.python.org/3/library/logging.config.html#logging-config-
    dictschema
    This dictionary is merged with the base logging configuration which defines
    the following:
    * A logging formatter intended for interactive use called
      ``console``.
    * A logging handler that writes to stderr called
      ``console`` which uses the formatter ``console``.
    * A logger with the name of this application set to ``DEBUG``
      level.
    This example adds a new handler that writes to a file:
    .. code-block:: python
       c.Application.logging_configuration = {
           'handlers': {
               'file': {
                   'class': 'logging.FileHandler',
                   'level': 'DEBUG',
                   'filename': '<path/to/file>',
               }
           },
           'loggers': {
               '<application-name>': {
                   'level': 'DEBUG',
                   # NOTE: if you don't list the default "console"
                   # handler here then it will be disabled
                   'handlers': ['console', 'file'],
               },
           }
       }
    Default: {}
--MyTool.provenance_log=<Path>
    Default: traitlets.Undefined
--MyTool.quiet=<Bool>
    Default: False
--MyTool.show_config=<Bool>
    Instead of starting the Application, dump configuration to stdout
    Default: False
--MyTool.show_config_json=<Bool>
    Instead of starting the Application, dump configuration to stdout (as JSON)
    Default: False

MyComponent(Component) options
------------------------------
--MyComponent.value=<Int>
    Value to use
    Default: -1

AdvancedComponent(Component) options
------------------------------------
--AdvancedComponent.infile=<Path>
    input file name
    Default: traitlets.Undefined
--AdvancedComponent.outfile=<Path>
    output file name
    Default: traitlets.Undefined
--AdvancedComponent.value1=<Int>
    Value to use
    Default: -1

SecondaryMyComponent(MyComponent) options
-----------------------------------------
--SecondaryMyComponent.value=<Int>
    Value to use
    Default: -1

TelescopeWiseComponent(TelescopeComponent) options
--------------------------------------------------
--TelescopeWiseComponent.param=<floattelescopeparameter-item-1>...
    Something configurable with telescope patterns
    Default: [('type', '*', 5.0)]

Run the tool

here we pass in argv since it is a Notebook, but if argv is not specified it’s read from sys.argv, so the following is the same as running:

mytool --log_level=INFO --infile gamma_test.simtel.gz --iterations=3

As Tools are intended to be exectutables, they are raising SystemExit on exit. Here, we use them to demonstrate how it would work, so we catch the SystemExit.

[12]:
try:
    tool.run(argv=['--infile',  str(GAMMA_FILE), '--outfile', 'out.csv'])
except SystemExit as e:
    assert e.code == 0, f'Tool returned with error status {e}'
Shutting down.
[13]:
tool.log_format = "%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s"


try:
    tool.run(argv=['--log-level','INFO','--infile', str(GAMMA_FILE), '--outfile', 'out.csv', '--iterations','3'])
except SystemExit as e:
    assert e.code == 0, f'Tool returned with error status {e}'
Shutting down.

here we change the log-level to DEBUG:

[14]:
try:
    tool.run(argv=['--log-level','DEBUG','--infile', str(GAMMA_FILE), '--outfile', 'out.csv'])
except SystemExit as e:
    assert e.code == 0, f'Tool returned with error status {e}'
2022-05-19 07:36:16,273 DEBUG [ctapipe.core.provenance] (provenance.start_activity): started activity: mytool
2022-05-19 07:36:16,277 DEBUG [ctapipe.core.traits] (traits.attach_subarray): argument '*' matched: ['LST_LST_LSTCam', 'MST_MST_NectarCam']
Shutting down.
2022-05-19 07:36:16,586 DEBUG [ctapipe.core.provenance] (provenance.finish_activity): finished activity: mytool

you can also set parameters directly in the class, rather than using the argument/configfile parser. This is useful if you are calling the Tool from a script rather than the command-line

[15]:
tool.iterations = 1
tool.log_level = 0

try:
    tool.run(['--infile', str(GAMMA_FILE), '--outfile', 'out.csv'])
except SystemExit as e:
    assert e.code == 0, f'Tool returned with error status {e}'
2022-05-19 07:36:16,623 DEBUG [ctapipe.core.provenance] (provenance.start_activity): started activity: mytool
2022-05-19 07:36:16,627 DEBUG [ctapipe.core.traits] (traits.attach_subarray): argument '*' matched: ['LST_LST_LSTCam', 'MST_MST_NectarCam']
Shutting down.
2022-05-19 07:36:16,936 DEBUG [ctapipe.core.provenance] (provenance.finish_activity): finished activity: mytool

see what happens when a value is set that is not of the correct type:

[16]:
try:
    tool.iterations = "badval"
except TraitError as E:
    print("bad value:",E)
except SystemExit as e:
    assert e.code == 0, f'Tool returned with error status {e}'
bad value: The 'iterations' trait of a MyTool instance expected an int, not the str 'badval'.

Example of what happens when you change a parameter that is being “observed” in a class. It’s handler is called:

[17]:
tool.advanced.outfile = "Another.txt"
Outfile was changed to '{'name': 'outfile', 'old': PosixPath('/home/runner/work/ctapipe/ctapipe/docs/examples/out.csv'), 'new': PosixPath('/home/runner/work/ctapipe/ctapipe/docs/examples/Another.txt'), 'owner': <__main__.AdvancedComponent object at 0x7fcc5c4ae340>, 'type': 'change'}'

we see that the handler for outfile was called, and it receive a change dict that shows the old and new values.

create a tool using a config file:

[18]:
tool2 = MyTool()
[19]:
try:
    tool2.run(argv=['--config', 'Tools.json'])
except SystemExit as e:
    assert e.code == 0, f'Tool returned with error status {e}'
2022-05-19 07:36:17,002 DEBUG [ctapipe.core.provenance] (provenance.start_activity): started activity: mytool
2022-05-19 07:36:17,006 DEBUG [ctapipe.core.traits] (traits.attach_subarray): argument '*' matched: ['LST_LST_LSTCam', 'MST_MST_NectarCam']
Shutting down.
2022-05-19 07:36:17,515 DEBUG [ctapipe.core.provenance] (provenance.finish_activity): finished activity: mytool
[20]:
print(tool2.advanced.infile)
/home/runner/work/ctapipe/ctapipe/docs/examples/something.txt
[21]:
print(tool2.config)
{'MyTool': {'config_files': ['Tools.json'], 'log_level': 'DEBUG'}, 'AdvancedComponent': {'infile': 'something.txt', 'outfile': 'foo.txt'}, 'attach_subarray': <LazyConfigValue {}>}
[22]:
tool2.is_setup
[22]:
True
[23]:
tool3 = MyTool()
[24]:
tool3.is_setup
[24]:
False
[25]:
tool3.initialize(argv=[])
[26]:
tool3.is_setup
[26]:
False
[27]:
tool3
[27]:
MyTool

do some things and stuff

config_files [] (default: traitlets.Undefined)
iterations 5 Number of times to run (default: 5)
log_config {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': , 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': , 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}} (default: traitlets.Undefined)
log_datefmt %Y-%m-%d %H:%M:%S The date format used by logging formatters for %(asctime)s (default: %Y-%m-%d %H:%M:%S)
log_file None Filename for the log (default: None)
log_file_level INFO Logging Level for File Logging (default: INFO)
log_format [%(name)s]%(highlevel)s %(message)s The Logging format template (default: [%(name)s]%(highlevel)s %(message)s)
log_level 30 Set the log level by value or name. (default: 30)
logging_config {} Configure additional log handlers. The default stderr logs handler is configured by the log_level, log_datefmt and log_format settings. This configuration can be used to configure additional handlers (e.g. to output the log to a file) or for finer control over the default handlers. If provided this should be a logging configuration dictionary, for more information see: https://docs.python.org/3/library/logging.config.html#logging-config-dictschema This dictionary is merged with the base logging configuration which defines the following: * A logging formatter intended for interactive use called ``console``. * A logging handler that writes to stderr called ``console`` which uses the formatter ``console``. * A logger with the name of this application set to ``DEBUG`` level. This example adds a new handler that writes to a file: .. code-block:: python c.Application.logging_configuration = { 'handlers': { 'file': { 'class': 'logging.FileHandler', 'level': 'DEBUG', 'filename': '', } }, 'loggers': { '': { 'level': 'DEBUG', # NOTE: if you don't list the default "console" # handler here then it will be disabled 'handlers': ['console', 'file'], }, } } (default: traitlets.Undefined)
provenance_log /home/runner/work/ctapipe/ctapipe/docs/examples/mytool.provenance.log (default: traitlets.Undefined)
quiet False (default: False)
show_config False Instead of starting the Application, dump configuration to stdout (default: False)
show_config_json False Instead of starting the Application, dump configuration to stdout (as JSON) (default: False)

Components: MyTool, MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent

[28]:
tool.setup()
tool
[28]:
MyTool

do some things and stuff

config_files [] (default: traitlets.Undefined)
iterations 3 Number of times to run (default: 5)
log_config {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': , 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': , 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}} (default: traitlets.Undefined)
log_datefmt %Y-%m-%d %H:%M:%S The date format used by logging formatters for %(asctime)s (default: %Y-%m-%d %H:%M:%S)
log_file None Filename for the log (default: None)
log_file_level INFO Logging Level for File Logging (default: INFO)
log_format %(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s The Logging format template (default: [%(name)s]%(highlevel)s %(message)s)
log_level 10 Set the log level by value or name. (default: 30)
logging_config {} Configure additional log handlers. The default stderr logs handler is configured by the log_level, log_datefmt and log_format settings. This configuration can be used to configure additional handlers (e.g. to output the log to a file) or for finer control over the default handlers. If provided this should be a logging configuration dictionary, for more information see: https://docs.python.org/3/library/logging.config.html#logging-config-dictschema This dictionary is merged with the base logging configuration which defines the following: * A logging formatter intended for interactive use called ``console``. * A logging handler that writes to stderr called ``console`` which uses the formatter ``console``. * A logger with the name of this application set to ``DEBUG`` level. This example adds a new handler that writes to a file: .. code-block:: python c.Application.logging_configuration = { 'handlers': { 'file': { 'class': 'logging.FileHandler', 'level': 'DEBUG', 'filename': '', } }, 'loggers': { '': { 'level': 'DEBUG', # NOTE: if you don't list the default "console" # handler here then it will be disabled 'handlers': ['console', 'file'], }, } } (default: traitlets.Undefined)
provenance_log /home/runner/work/ctapipe/ctapipe/docs/examples/mytool.provenance.log (default: traitlets.Undefined)
quiet False (default: False)
show_config False Instead of starting the Application, dump configuration to stdout (default: False)
show_config_json False Instead of starting the Application, dump configuration to stdout (as JSON) (default: False)

Components: MyTool, MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent

[29]:
tool.comp2
[29]:
SecondaryMyComponent

Undocumented!

value -1 Value to use (default: -1)

Getting the configuration of an instance

[30]:
tool.get_current_config()
[30]:
{'MyTool': {'config_files': [],
  'iterations': 3,
  'log_config': {'version': 1,
   'disable_existing_loggers': False,
   'formatters': {'file': {'()': ctapipe.core.logging.PlainFormatter,
     'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'},
    'console': {'()': ctapipe.core.logging.ColoredFormatter,
     'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}},
   'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler',
     'formatter': 'console',
     'stream': 'ext://sys.stderr',
     'level': 30},
    'console': {'class': 'logging.StreamHandler',
     'formatter': 'console',
     'stream': 'ext://sys.stderr',
     'level': 'NOTSET'}},
   'loggers': {'ctapipe': {'level': 30,
     'handlers': ['ctapipe-console'],
     'propagate': False}}},
  'log_datefmt': '%Y-%m-%d %H:%M:%S',
  'log_file': None,
  'log_file_level': 'INFO',
  'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s',
  'log_level': 10,
  'logging_config': {},
  'provenance_log': PosixPath('/home/runner/work/ctapipe/ctapipe/docs/examples/mytool.provenance.log'),
  'quiet': False,
  'show_config': False,
  'show_config_json': False,
  'MyComponent': {'value': -1},
  'SecondaryMyComponent': {'value': -1},
  'TelescopeWiseComponent': {'param': TelescopePatternList([('type',
                          '*',
                          5.0)])},
  'AdvancedComponent': {'infile': PosixPath('/home/runner/.cache/ctapipe/cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/gamma_test.simtel.gz'),
   'outfile': PosixPath('/home/runner/work/ctapipe/ctapipe/docs/examples/out.csv'),
   'value1': -1,
   'MyComponent': {'value': -1}}}}
[31]:
tool.iterations = 12
tool.get_current_config()
[31]:
{'MyTool': {'config_files': [],
  'iterations': 12,
  'log_config': {'version': 1,
   'disable_existing_loggers': False,
   'formatters': {'file': {'()': ctapipe.core.logging.PlainFormatter,
     'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'},
    'console': {'()': ctapipe.core.logging.ColoredFormatter,
     'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}},
   'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler',
     'formatter': 'console',
     'stream': 'ext://sys.stderr',
     'level': 30},
    'console': {'class': 'logging.StreamHandler',
     'formatter': 'console',
     'stream': 'ext://sys.stderr',
     'level': 'NOTSET'}},
   'loggers': {'ctapipe': {'level': 30,
     'handlers': ['ctapipe-console'],
     'propagate': False}}},
  'log_datefmt': '%Y-%m-%d %H:%M:%S',
  'log_file': None,
  'log_file_level': 'INFO',
  'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s',
  'log_level': 10,
  'logging_config': {},
  'provenance_log': PosixPath('/home/runner/work/ctapipe/ctapipe/docs/examples/mytool.provenance.log'),
  'quiet': False,
  'show_config': False,
  'show_config_json': False,
  'MyComponent': {'value': -1},
  'SecondaryMyComponent': {'value': -1},
  'TelescopeWiseComponent': {'param': TelescopePatternList([('type',
                          '*',
                          5.0)])},
  'AdvancedComponent': {'infile': PosixPath('/home/runner/.cache/ctapipe/cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/gamma_test.simtel.gz'),
   'outfile': PosixPath('/home/runner/work/ctapipe/ctapipe/docs/examples/out.csv'),
   'value1': -1,
   'MyComponent': {'value': -1}}}}

Writing a Sample Config File

[32]:
print(tool.generate_config_file())
# Configuration file for mytool.

#------------------------------------------------------------------------------
# Application(SingletonConfigurable) configuration
#------------------------------------------------------------------------------
## This is an application.

## The date format used by logging formatters for %(asctime)s
#  Default: '%Y-%m-%d %H:%M:%S'
# c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S'

## The Logging format template
#  Default: '[%(name)s]%(highlevel)s %(message)s'
# c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s'

## Set the log level by value or name.
#  Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
#  Default: 30
# c.Application.log_level = 30

## Configure additional log handlers.
#
#  The default stderr logs handler is configured by the log_level, log_datefmt
#  and log_format settings.
#
#  This configuration can be used to configure additional handlers (e.g. to
#  output the log to a file) or for finer control over the default handlers.
#
#  If provided this should be a logging configuration dictionary, for more
#  information see:
#  https://docs.python.org/3/library/logging.config.html#logging-config-
#  dictschema
#
#  This dictionary is merged with the base logging configuration which defines
#  the following:
#
#  * A logging formatter intended for interactive use called
#    ``console``.
#  * A logging handler that writes to stderr called
#    ``console`` which uses the formatter ``console``.
#  * A logger with the name of this application set to ``DEBUG``
#    level.
#
#  This example adds a new handler that writes to a file:
#
#  .. code-block:: python
#
#     c.Application.logging_configuration = {
#         'handlers': {
#             'file': {
#                 'class': 'logging.FileHandler',
#                 'level': 'DEBUG',
#                 'filename': '<path/to/file>',
#             }
#         },
#         'loggers': {
#             '<application-name>': {
#                 'level': 'DEBUG',
#                 # NOTE: if you don't list the default "console"
#                 # handler here then it will be disabled
#                 'handlers': ['console', 'file'],
#             },
#         }
#     }
#  Default: {}
# c.Application.logging_config = {}

## Instead of starting the Application, dump configuration to stdout
#  Default: False
# c.Application.show_config = False

## Instead of starting the Application, dump configuration to stdout (as JSON)
#  Default: False
# c.Application.show_config_json = False

#------------------------------------------------------------------------------
# Tool(Application) configuration
#------------------------------------------------------------------------------
## This is an application.

#  Default: []
# c.Tool.config_files = []

#  Default: {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': <class 'ctapipe.core.logging.PlainFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': <class 'ctapipe.core.logging.ColoredFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}}
# c.Tool.log_config = {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': <class 'ctapipe.core.logging.PlainFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': <class 'ctapipe.core.logging.ColoredFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}}

## The date format used by logging formatters for %(asctime)s
#  See also: Application.log_datefmt
# c.Tool.log_datefmt = '%Y-%m-%d %H:%M:%S'

## Filename for the log
#  Default: None
# c.Tool.log_file = None

## Logging Level for File Logging
#  Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
#  Default: 'INFO'
# c.Tool.log_file_level = 'INFO'

## The Logging format template
#  See also: Application.log_format
# c.Tool.log_format = '[%(name)s]%(highlevel)s %(message)s'

## Set the log level by value or name.
#  See also: Application.log_level
# c.Tool.log_level = 30

##
#  See also: Application.logging_config
# c.Tool.logging_config = {}

#  Default: traitlets.Undefined
# c.Tool.provenance_log = traitlets.Undefined

#  Default: False
# c.Tool.quiet = False

## Instead of starting the Application, dump configuration to stdout
#  See also: Application.show_config
# c.Tool.show_config = False

## Instead of starting the Application, dump configuration to stdout (as JSON)
#  See also: Application.show_config_json
# c.Tool.show_config_json = False

#------------------------------------------------------------------------------
# MyTool(Tool) configuration
#------------------------------------------------------------------------------
#  See also: Tool.config_files
# c.MyTool.config_files = []

## Number of times to run
#  Default: 5
# c.MyTool.iterations = 5

#  See also: Tool.log_config
# c.MyTool.log_config = {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': <class 'ctapipe.core.logging.PlainFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': <class 'ctapipe.core.logging.ColoredFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}}

## The date format used by logging formatters for %(asctime)s
#  See also: Application.log_datefmt
# c.MyTool.log_datefmt = '%Y-%m-%d %H:%M:%S'

## Filename for the log
#  See also: Tool.log_file
# c.MyTool.log_file = None

## Logging Level for File Logging
#  See also: Tool.log_file_level
# c.MyTool.log_file_level = 'INFO'

## The Logging format template
#  See also: Application.log_format
# c.MyTool.log_format = '[%(name)s]%(highlevel)s %(message)s'

## Set the log level by value or name.
#  See also: Application.log_level
# c.MyTool.log_level = 30

##
#  See also: Application.logging_config
# c.MyTool.logging_config = {}

#  See also: Tool.provenance_log
# c.MyTool.provenance_log = traitlets.Undefined

#  See also: Tool.quiet
# c.MyTool.quiet = False

## Instead of starting the Application, dump configuration to stdout
#  See also: Application.show_config
# c.MyTool.show_config = False

## Instead of starting the Application, dump configuration to stdout (as JSON)
#  See also: Application.show_config_json
# c.MyTool.show_config_json = False

#------------------------------------------------------------------------------
# MyComponent(Component) configuration
#------------------------------------------------------------------------------
## A Component that does stuff

## Value to use
#  Default: -1
# c.MyComponent.value = -1

#------------------------------------------------------------------------------
# AdvancedComponent(Component) configuration
#------------------------------------------------------------------------------
## An advanced technique

## input file name
#  Default: traitlets.Undefined
# c.AdvancedComponent.infile = traitlets.Undefined

## output file name
#  Default: traitlets.Undefined
# c.AdvancedComponent.outfile = traitlets.Undefined

## Value to use
#  Default: -1
# c.AdvancedComponent.value1 = -1

#------------------------------------------------------------------------------
# SecondaryMyComponent(MyComponent) configuration
#------------------------------------------------------------------------------
## Value to use
#  See also: MyComponent.value
# c.SecondaryMyComponent.value = -1

#------------------------------------------------------------------------------
# TelescopeWiseComponent(TelescopeComponent) configuration
#------------------------------------------------------------------------------
## a component that contains parameters that are per-telescope configurable

## Something configurable with telescope patterns
#  Default: [('type', '*', 5.0)]
# c.TelescopeWiseComponent.param = [('type', '*', 5.0)]

[ ]: