Introduction

lstchain is the analysis library for the observed and simulated LST-1 data.

Dependencies

lstchain heavily depends on ctapipe

Data levels

Level	Description	File Format
R0	Uncalibrated RAW waveforms from the camera	ZFITS
R1	Calibrated waveforms from the camera
DL1a	Integrated charge and peak position of the waveform	HDF5
DL1b	Image parameters (width, length, intensity, etc.)	HDF5
DL2	Event parameters (energy, direction, time, etc.)	HDF5
DL3	Lists of reconstructed events after event selection with corresponding IRFs (AEFF, EDISP, PSF, etc.)	FITS

Analysis steps

These individuals steps can be run one by one but are also integrated in more complete workflows presented in LST-1 data analysis workflow.

R1 to DL1

MC data

Use lstchain.scripts.lstchain_mc_r0_to_dl1.

For more information, try --help or see the API Docs.

Real data

Use lstchain.scripts.lstchain_data_r0_to_dl1.

For more information, try --help or see the API Docs.

DL1 to DL1a and DL1b

If you already have a DL1 file containing images and parameters (DL1a and DL1b), you can recalculate the parameters using a different cleaning by using lstchain.scripts.lstchain_dl1ab.

For more information, try --help or see the API Docs.

Configuration file

Here is an example configuration file for this step.

{
  "source_config" : {
    "EventSource": {
      "allowed_tels": [1],
      "max_events": null
    },
    "LSTEventSource": {
      "default_trigger_type": "ucts",
      "allowed_tels": [1],
      "min_flatfield_adc": 3000,
      "min_flatfield_pixel_fraction": 0.8,
      "calibrate_flatfields_and_pedestals": false,
      "EventTimeCalculator": {
        "dragon_reference_counter": null,
        "dragon_reference_time": null
      },
      "PointingSource":{
        "drive_report_path": null
      },
      "LSTR0Corrections":{
        "calib_scale_high_gain":1.088,
        "calib_scale_low_gain":1.004,
        "drs4_pedestal_path": null,
        "calibration_path": null,
        "drs4_time_calibration_path": null
      }
    }
  },

  "events_filters": {
    "intensity": [0, Infinity],
    "width": [0, Infinity],
    "length": [0, Infinity],
    "wl": [0, Infinity],
    "r": [0, Infinity],
    "leakage_intensity_width_2": [0, Infinity]
  },
  "n_training_events": {
    "gamma_regressors": 1.0,
    "gamma_tmp_regressors": 0.8,
    "gamma_classifier": 0.2,
    "proton_classifier": 1.0
  },

  "tailcut": {
    "picture_thresh":8,
    "boundary_thresh":4,
    "keep_isolated_pixels":false,
    "min_number_picture_neighbors":2,
    "use_only_main_island":false,
    "delta_time": 2
  },
  "tailcuts_clean_with_pedestal_threshold": {
    "picture_thresh":8,
    "boundary_thresh":4,
    "sigma":2.5,
    "keep_isolated_pixels":false,
    "min_number_picture_neighbors":2,
    "use_only_main_island":false,
    "delta_time": 2
  },
  "dynamic_cleaning": {
    "apply": true,
    "threshold": 267,
    "fraction_cleaning_intensity": 0.03
  },

  "random_forest_weight_settings": {
    "pointing_wise_weights": true
  },

  "random_forest_zd_interpolation": {
    "interpolate_energy": true,
    "interpolate_gammaness": true,
    "interpolate_direction": true
  },

  "random_forest_energy_regressor_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 50,
    "bootstrap": true,
    "criterion": "squared_error",
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false
  },

  "random_forest_disp_regressor_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 50,
    "bootstrap": true,
    "criterion": "squared_error",
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false
  },

  "random_forest_disp_classifier_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 100,
    "criterion": "gini",
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "bootstrap": true,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false,
    "class_weight": null
  },

  "random_forest_particle_classifier_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 100,
    "criterion": "gini",
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "bootstrap": true,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false,
    "class_weight": null
  },


  "energy_regression_features": [
    "log_intensity",
    "width",
    "length",
    "x",
    "y",
    "wl",
    "skewness",
    "kurtosis",
    "time_gradient",
    "leakage_intensity_width_2",
    "sin_az_tel",
    "alt_tel"
  ],

  "disp_method": "disp_norm_sign",

  "disp_regression_features": [
    "log_intensity",
    "width",
    "length",
    "wl",
    "skewness",
    "kurtosis",
    "time_gradient",
    "leakage_intensity_width_2",
    "sin_az_tel",
    "alt_tel"
  ],

  "disp_classification_features": [
    "log_intensity",
    "width",
    "length",
    "wl",
    "skewness",
    "kurtosis",
    "time_gradient",
    "leakage_intensity_width_2",
    "sin_az_tel",
    "alt_tel"
  ],

  "particle_classification_features": [
    "log_intensity",
    "width",
    "length",
    "x",
    "y",
    "wl",
    "signed_skewness",
    "kurtosis",
    "signed_time_gradient",
    "leakage_intensity_width_2",
    "log_reco_energy",
    "reco_disp_norm",
    "reco_disp_sign",
    "sin_az_tel",
    "alt_tel"
  ],

  "allowed_tels": [1],
  "write_pe_image": false,
  "mc_image_scaling_factor": 1,
  "image_extractor": "LocalPeakWindowSum",
  "image_extractor_for_muons": "GlobalPeakWindowSum",
  "CameraCalibrator": {
    "apply_waveform_time_shift": false
  },
  "time_sampling_correction_path": "default",
  "LocalPeakWindowSum":{
    "window_shift": 4,
    "window_width": 8,
    "apply_integration_correction": false
  },
  "GlobalPeakWindowSum":{
    "window_shift": 4,
    "window_width": 8,
    "apply_integration_correction": false
  },
  "timestamps_pointing":"ucts",

  "train_gamma_src_r_deg": [0, Infinity],

  "source_dependent": false,
  "mc_nominal_source_x_deg": 0.4,
  "mc_nominal_source_y_deg": 0.0,

  "volume_reducer":{
    "algorithm": null,
    "parameters": {
    }
  },
  "calibration_product": "LSTCalibrationCalculator",

  "LSTCalibrationCalculator":{
    "systematic_correction_path": null,
    "npe_median_cut_outliers": [-5,5],
    "squared_excess_noise_factor": 1.222,
    "flatfield_product": "FlasherFlatFieldCalculator",
    "pedestal_product": "PedestalIntegrator",
    "PedestalIntegrator":{
      "sample_size": 10000,
      "sample_duration":100000,
      "tel_id":1,
      "time_sampling_correction_path": null,
      "charge_median_cut_outliers": [-10,10],
      "charge_std_cut_outliers": [-10,10],
      "charge_product":"FixedWindowSum",
      "FixedWindowSum":{
        "window_shift": 6,
        "window_width":12,
        "peak_index": 18,
        "apply_integration_correction": false
      }
    },
    "FlasherFlatFieldCalculator":{
      "sample_size": 10000,
      "sample_duration":100000,
      "tel_id":1,
      "time_sampling_correction_path": null,
      "charge_product":"LocalPeakWindowSum",
      "charge_median_cut_outliers": [-0.9,2],
      "charge_std_cut_outliers": [-10,10],
      "time_cut_outliers": [2,38],
      "LocalPeakWindowSum":{
        "window_shift": 5,
        "window_width":12,
        "apply_integration_correction": false
      }
    }
  },
  "waveform_nsb_tuning":{
    "nsb_tuning": false,
    "nsb_tuning_rate_GHz": 0.15,
    "spe_location": null,
    "pre_computed_multiplicity": 10
  },
  "write_interleaved_events":{
    "DataWriter": {
      "overwrite":  true,
      "write_images": false,
      "write_parameters": false,
      "write_waveforms": true,
      "transform_waveform": true,
      "waveform_dtype": "uint16",
      "waveform_offset": 400,
      "waveform_scale": 80
    }
  }
}

DL1 to DL2

Use lstchain.tools.lstchain_dl1_to_dl2 for real data and MC.

For more information, try --help or see the API Docs.

Configuration file

Here is an example configuration file for this step.

{
  "source_config" : {
    "EventSource": {
      "allowed_tels": [1],
      "max_events": null
    },
    "LSTEventSource": {
      "default_trigger_type": "ucts",
      "allowed_tels": [1],
      "min_flatfield_adc": 3000,
      "min_flatfield_pixel_fraction": 0.8,
      "calibrate_flatfields_and_pedestals": false,
      "EventTimeCalculator": {
        "dragon_reference_counter": null,
        "dragon_reference_time": null
      },
      "PointingSource":{
        "drive_report_path": null
      },
      "LSTR0Corrections":{
        "calib_scale_high_gain":1.088,
        "calib_scale_low_gain":1.004,
        "drs4_pedestal_path": null,
        "calibration_path": null,
        "drs4_time_calibration_path": null
      }
    }
  },

  "events_filters": {
    "intensity": [0, Infinity],
    "width": [0, Infinity],
    "length": [0, Infinity],
    "wl": [0, Infinity],
    "r": [0, Infinity],
    "leakage_intensity_width_2": [0, Infinity]
  },
  "n_training_events": {
    "gamma_regressors": 1.0,
    "gamma_tmp_regressors": 0.8,
    "gamma_classifier": 0.2,
    "proton_classifier": 1.0
  },

  "tailcut": {
    "picture_thresh":8,
    "boundary_thresh":4,
    "keep_isolated_pixels":false,
    "min_number_picture_neighbors":2,
    "use_only_main_island":false,
    "delta_time": 2
  },
  "tailcuts_clean_with_pedestal_threshold": {
    "picture_thresh":8,
    "boundary_thresh":4,
    "sigma":2.5,
    "keep_isolated_pixels":false,
    "min_number_picture_neighbors":2,
    "use_only_main_island":false,
    "delta_time": 2
  },
  "dynamic_cleaning": {
    "apply": true,
    "threshold": 267,
    "fraction_cleaning_intensity": 0.03
  },

  "random_forest_weight_settings": {
    "pointing_wise_weights": true
  },

  "random_forest_zd_interpolation": {
    "interpolate_energy": true,
    "interpolate_gammaness": true,
    "interpolate_direction": true
  },

  "random_forest_energy_regressor_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 50,
    "bootstrap": true,
    "criterion": "squared_error",
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false
  },

  "random_forest_disp_regressor_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 50,
    "bootstrap": true,
    "criterion": "squared_error",
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false
  },

  "random_forest_disp_classifier_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 100,
    "criterion": "gini",
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "bootstrap": true,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false,
    "class_weight": null
  },

  "random_forest_particle_classifier_args": {
    "max_depth": 30,
    "min_samples_leaf": 10,
    "n_jobs": -1,
    "n_estimators": 100,
    "criterion": "gini",
    "min_samples_split": 10,
    "min_weight_fraction_leaf": 0.0,
    "max_features": 1.0,
    "max_leaf_nodes": null,
    "min_impurity_decrease": 0.0,
    "bootstrap": true,
    "oob_score": false,
    "random_state": 42,
    "warm_start": false,
    "class_weight": null
  },


  "energy_regression_features": [
    "log_intensity",
    "width",
    "length",
    "x",
    "y",
    "wl",
    "skewness",
    "kurtosis",
    "time_gradient",
    "leakage_intensity_width_2",
    "sin_az_tel",
    "alt_tel"
  ],

  "disp_method": "disp_norm_sign",

  "disp_regression_features": [
    "log_intensity",
    "width",
    "length",
    "wl",
    "skewness",
    "kurtosis",
    "time_gradient",
    "leakage_intensity_width_2",
    "sin_az_tel",
    "alt_tel"
  ],

  "disp_classification_features": [
    "log_intensity",
    "width",
    "length",
    "wl",
    "skewness",
    "kurtosis",
    "time_gradient",
    "leakage_intensity_width_2",
    "sin_az_tel",
    "alt_tel"
  ],

  "particle_classification_features": [
    "log_intensity",
    "width",
    "length",
    "x",
    "y",
    "wl",
    "signed_skewness",
    "kurtosis",
    "signed_time_gradient",
    "leakage_intensity_width_2",
    "log_reco_energy",
    "reco_disp_norm",
    "reco_disp_sign",
    "sin_az_tel",
    "alt_tel"
  ],

  "allowed_tels": [1],
  "write_pe_image": false,
  "mc_image_scaling_factor": 1,
  "image_extractor": "LocalPeakWindowSum",
  "image_extractor_for_muons": "GlobalPeakWindowSum",
  "CameraCalibrator": {
    "apply_waveform_time_shift": false
  },
  "time_sampling_correction_path": "default",
  "LocalPeakWindowSum":{
    "window_shift": 4,
    "window_width": 8,
    "apply_integration_correction": false
  },
  "GlobalPeakWindowSum":{
    "window_shift": 4,
    "window_width": 8,
    "apply_integration_correction": false
  },
  "timestamps_pointing":"ucts",

  "train_gamma_src_r_deg": [0, Infinity],

  "source_dependent": false,
  "mc_nominal_source_x_deg": 0.4,
  "mc_nominal_source_y_deg": 0.0,

  "volume_reducer":{
    "algorithm": null,
    "parameters": {
    }
  },
  "calibration_product": "LSTCalibrationCalculator",

  "LSTCalibrationCalculator":{
    "systematic_correction_path": null,
    "npe_median_cut_outliers": [-5,5],
    "squared_excess_noise_factor": 1.222,
    "flatfield_product": "FlasherFlatFieldCalculator",
    "pedestal_product": "PedestalIntegrator",
    "PedestalIntegrator":{
      "sample_size": 10000,
      "sample_duration":100000,
      "tel_id":1,
      "time_sampling_correction_path": null,
      "charge_median_cut_outliers": [-10,10],
      "charge_std_cut_outliers": [-10,10],
      "charge_product":"FixedWindowSum",
      "FixedWindowSum":{
        "window_shift": 6,
        "window_width":12,
        "peak_index": 18,
        "apply_integration_correction": false
      }
    },
    "FlasherFlatFieldCalculator":{
      "sample_size": 10000,
      "sample_duration":100000,
      "tel_id":1,
      "time_sampling_correction_path": null,
      "charge_product":"LocalPeakWindowSum",
      "charge_median_cut_outliers": [-0.9,2],
      "charge_std_cut_outliers": [-10,10],
      "time_cut_outliers": [2,38],
      "LocalPeakWindowSum":{
        "window_shift": 5,
        "window_width":12,
        "apply_integration_correction": false
      }
    }
  },
  "waveform_nsb_tuning":{
    "nsb_tuning": false,
    "nsb_tuning_rate_GHz": 0.15,
    "spe_location": null,
    "pre_computed_multiplicity": 10
  },
  "write_interleaved_events":{
    "DataWriter": {
      "overwrite":  true,
      "write_images": false,
      "write_parameters": false,
      "write_waveforms": true,
      "transform_waveform": true,
      "waveform_dtype": "uint16",
      "waveform_offset": 400,
      "waveform_scale": 80
    }
  }
}

DL2 to DL3

For a quick look into the data and perform \({\theta}^2/{\alpha}\) plots from DL2 files, you can use the notebook:

https://github.com/cta-observatory/cta-lstchain/blob/main/notebooks/explore_DL2.ipynb

or the script: lstchain_significance_calculation

IRF creation

To write IRF files, you should use lstchain.tools.lstchain_create_irf_files.

For more information, try --help or see the API Docs.

Here is an example configuration file for the IRF creation step.

{
  "EventSelector": {
    "filters": {
      "intensity": [50, Infinity],
      "width": [0, Infinity],
      "length": [0, Infinity],
      "r": [0, 1],
      "wl": [0.01, 1],
      "leakage_intensity_width_2": [0, 1],
      "event_type": [32, 32]
    }
  },
  "DL3Cuts": {
    "min_event_p_en_bin": 100,
    "global_gh_cut": 0.7,
    "gh_efficiency": 0.7,
    "min_gh_cut": 0.1,
    "max_gh_cut": 0.98,
    "global_alpha_cut": 10,
    "global_theta_cut": 0.2,
    "theta_containment": 0.7,
    "alpha_containment": 0.7,
    "min_theta_cut": 0.1,
    "max_theta_cut": 0.32,
    "fill_theta_cut": 0.32,
    "min_alpha_cut": 1,
    "max_alpha_cut": 20,
    "fill_alpha_cut": 20,
    "allowed_tels": [1]
  },
  "DataBinning": {
    "true_energy_min": 0.005,
    "true_energy_max": 500,
    "true_energy_n_bins": 25,
    "scale_true_energy": 1.0,
    "reco_energy_min": 0.005,
    "reco_energy_max": 500,
    "reco_energy_n_bins": 25,
    "energy_migration_min": 0.2,
    "energy_migration_max": 5,
    "energy_migration_n_bins": 30,
    "fov_offset_min": 0.1,
    "fov_offset_max": 1.1,
    "fov_offset_n_edges": 9,
    "bkg_fov_offset_min": 0,
    "bkg_fov_offset_max": 10,
    "bkg_fov_offset_n_edges": 21,
    "source_offset_min": 0,
    "source_offset_max": 1,
    "source_offset_n_edges": 101
  }
}

Event list creation

To write DL3 files, you should use:

• lstchain.tools.lstchain_create_dl3_file

• lstchain.tools.lstchain_create_dl3_index_files

For more information, try --help or see the API Docs.

You should use the same configuration file used for the IRF creation (hence you have the same cuts).

Post DL3 analysis

You can analyze the resulting DL3 files using Gammapy https://docs.gammapy.org/.

There is a notebook in cta-lstchain/notebooks that shows how to use Gammapy to perform a 1D on-off analysis for point-like sources.

https://github.com/cta-observatory/cta-lstchain/blob/main/notebooks/post_DL3_analysis.ipynb

Other analyses require additional packages. For example, for the 3D analysis of extended sources and creating sky maps, you can create a background model using one of the following packages:

• https://github.com/mdebony/BAccMod

• https://github.com/cta-observatory/pybkgmodel

For doing pulsar analysis we suggest using:

• https://github.com/cta-observatory/PulsarTimingAnalysis