Skip to content

Arase

el_paso.recipes.arase.get_arase_orbit_variables.get_arase_orbit_level_2_variables 

get_arase_orbit_level_2_variables

Download and extract Arase Level 2 orbit (definitive) data.

Downloads the daily Arase orb/def Level 2 CDF files covering the requested time range (skipping files that already exist) and extracts the epoch and SM position variables from them.

Parameters:

Name Type Description Default
start_time datetime

Start of the time range to retrieve data for.

 required
end_time datetime

End of the time range to retrieve data for.

 required
raw_data_path str | Path

Directory where the downloaded raw data files are stored. Defaults to ".".

 '.'

Returns:

Type Description
dict[str, Variable]

dict[str, ep.Variable]: A dictionary containing the extracted "Epoch" and "pos_sm" variables.
Source code in el_paso/recipes/arase/get_arase_orbit_variables.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def get_arase_orbit_level_2_variables(
    start_time: datetime, end_time: datetime, raw_data_path: str | Path = "."
) -> dict[str, ep.Variable]:
    """Download and extract Arase Level 2 orbit (definitive) data.

    Downloads the daily Arase ``orb/def`` Level 2 CDF files covering the requested time range
    (skipping files that already exist) and extracts the epoch and SM position variables from them.

    Args:
        start_time (datetime): Start of the time range to retrieve data for.
        end_time (datetime): End of the time range to retrieve data for.
        raw_data_path (str | Path, optional): Directory where the downloaded raw data files are
                                            stored. Defaults to ".".

    Returns:
        dict[str, ep.Variable]: A dictionary containing the extracted "Epoch" and "pos_sm" variables.
    """
    raw_data_path = Path(raw_data_path)

    file_name_stem = "erg_orb_l2_YYYYMMDD_.{3}.cdf"
    url = "https://ergsc.isee.nagoya-u.ac.jp/data/ergsc/satellite/erg/orb/def/YYYY/"

    ep.download(
        start_time,
        end_time,
        save_path=raw_data_path,
        download_url=url,
        file_name_stem=file_name_stem,
        file_cadence="daily",
        method="request",
        skip_existing=True,
    )

    extraction_infos = [
        ep.ExtractionInfo(
            result_key="Epoch",
            name_or_column="epoch",
            unit=ep.units.tt2000,
        ),
        ep.ExtractionInfo(
            result_key="pos_sm",
            name_or_column="pos_sm",
            unit=u.dimensionless_unscaled,
        ),
    ]

    variables = ep.extract_variables_from_files(
        start_time,
        end_time,
        "daily",
        data_path=raw_data_path,
        file_name_stem=file_name_stem,
        extraction_infos=extraction_infos,
    )
    return variables

el_paso.recipes.arase.get_arase_orbit_variables.get_arase_orbit_level_3_variables 

get_arase_orbit_level_3_variables

Download and extract Arase Level 3 orbit data for a given magnetic field model.

Downloads the daily Arase orb/l3 Level 3 CDF files corresponding to the given mag_field model (skipping files that already exist), extracts the epoch, local and equatorial magnetic field magnitude, Lm, Lstar and equatorial position variables, truncates all variables to the requested time range, and derives "MLT" and "R0" variables from the equatorial position.

Parameters:

Name Type Description Default
start_time datetime

Start of the time range to retrieve data for.

 required
end_time datetime

End of the time range to retrieve data for.

 required
mag_field Literal['OP77Q', 'T89', 'TS04']

The magnetic field model whose Level 3 orbit data should be downloaded and extracted.

 required
raw_data_path str | Path

Directory where the downloaded raw data files are stored. Defaults to ".".

 '.'

Returns:

Type Description
dict[str, Variable]

dict[str, ep.Variable]: A dictionary containing the extracted and derived variables ("Epoch", "B_local", "B_eq", "Lm", "Lstar", "MLT", "R0").

Raises:

Type Description
ValueError

If mag_field is not one of "OP77Q", "T89" or "TS04".
Source code in el_paso/recipes/arase/get_arase_orbit_variables.py 
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def get_arase_orbit_level_3_variables(
    start_time: datetime,
    end_time: datetime,
    mag_field: Literal["OP77Q", "T89", "TS04"],
    raw_data_path: str | Path = ".",
) -> dict[str, ep.Variable]:
    """Download and extract Arase Level 3 orbit data for a given magnetic field model.

    Downloads the daily Arase ``orb/l3`` Level 3 CDF files corresponding to the given `mag_field`
    model (skipping files that already exist), extracts the epoch, local and equatorial magnetic
    field magnitude, Lm, Lstar and equatorial position variables, truncates all variables to the
    requested time range, and derives "MLT" and "R0" variables from the equatorial position.

    Args:
        start_time (datetime): Start of the time range to retrieve data for.
        end_time (datetime): End of the time range to retrieve data for.
        mag_field (Literal["OP77Q", "T89", "TS04"]): The magnetic field model whose Level 3 orbit
                                                    data should be downloaded and extracted.
        raw_data_path (str | Path, optional): Directory where the downloaded raw data files are
                                            stored. Defaults to ".".

    Returns:
        dict[str, ep.Variable]: A dictionary containing the extracted and derived variables
                            ("Epoch", "B_local", "B_eq", "Lm", "Lstar", "MLT", "R0").

    Raises:
        ValueError: If `mag_field` is not one of "OP77Q", "T89" or "TS04".
    """
    raw_data_path = Path(raw_data_path)

    match mag_field:
        case "OP77Q":
            mag_field_label = "opq"
        case "T89":
            mag_field_label = "t89"
        case "TS04":
            mag_field_label = "ts04"
        case _:
            msg = "Encountered invalid mag field for Arase orbit: {mag_field}!"
            raise ValueError(msg)

    url = f"https://ergsc.isee.nagoya-u.ac.jp/data/ergsc/satellite/erg/orb/l3/{mag_field_label}/YYYY/MM/"

    if mag_field_label == "opq":
        mag_field_label = "op"
    file_name_stem = "erg_orb_l3_" + mag_field_label + "_YYYYMMDD_.{3}.cdf"

    ep.download(
        start_time,
        end_time,
        save_path=raw_data_path,
        download_url=url,
        file_name_stem=file_name_stem,
        file_cadence="daily",
        method="request",
        skip_existing=True,
    )

    match mag_field:
        case "OP77Q":
            mag_field_label = "op"
        case "T89":
            mag_field_label = "t89"
        case "TS04":
            mag_field_label = "TS04"
        case _:
            msg = "Encountered invalid mag field for Arase orbit: {mag_field}!"
            raise ValueError(msg)

    extraction_infos = [
        ep.ExtractionInfo(
            result_key="Epoch",
            name_or_column="epoch",
            unit=ep.units.tt2000,
        ),
        ep.ExtractionInfo(
            result_key="B_local",
            name_or_column=f"pos_blocal_{mag_field_label}",
            unit=u.nT,
        ),
        ep.ExtractionInfo(
            result_key="B_eq",
            name_or_column=f"pos_beq_{mag_field_label}",
            unit=u.nT,
        ),
        ep.ExtractionInfo(
            result_key="Lm",
            name_or_column=f"pos_lmc_{mag_field_label}",
            unit=u.dimensionless_unscaled,
        ),
        ep.ExtractionInfo(
            result_key="Lstar",
            name_or_column=f"pos_lstar_{mag_field_label}",
            unit=u.dimensionless_unscaled,
        ),
        ep.ExtractionInfo(
            result_key="pos_eq",
            name_or_column=f"pos_eq_{mag_field_label}",
            unit=u.dimensionless_unscaled,
        ),
    ]

    variables = ep.extract_variables_from_files(
        start_time,
        end_time,
        "daily",
        data_path=raw_data_path,
        file_name_stem=file_name_stem,
        extraction_infos=extraction_infos,
    )

    for key in variables:
        if key == "Epoch":
            continue
        variables[key].truncate(variables["Epoch"], start_time, end_time)
    variables["Epoch"].truncate(variables["Epoch"], start_time, end_time)

    mlt_data = variables["pos_eq"].get_data()[:, 1]
    R_data = variables["pos_eq"].get_data()[:, 0]

    variables["MLT"] = ep.Variable(data=mlt_data, original_unit=u.hour)
    variables["R0"] = ep.Variable(data=R_data, original_unit=ep.units.RE)

    del variables["pos_eq"]

    return variables

el_paso.recipes.arase.process_arase_mepe.process_arase_mepe 

process_arase_mepe

Process Arase MEP-e Level 3 electron flux data and save derived products.

Downloads the corresponding Arase orbit data (Level 3 if use_level_3_orbit_data is True, otherwise Level 2) and the daily Arase MEP-e Level 3 pitch-angle resolved flux (FEDU) CDF files for the requested time range, then sorts energies into ascending order, time-bins the flux, position and pitch-angle variables onto a common cadence, folds the pitch-angle distribution, applies a lower flux threshold, and computes the equatorial pitch angle and magnetic-field-related quantities (Lm, MLT, R0) either from the Level 3 orbit data directly or via IRBEM using the SM position from Level 2 orbit data. The resulting variables are saved using the requested saving strategy.

Parameters:

Name Type Description Default
start_time datetime

Start of the time range to process.

 required
end_time datetime

End of the time range to process.

 required
mag_field Literal['T89', 'TS04', 'OP77Q']

The magnetic field model used for the magnetic-field-related output variables.

 required
raw_data_path str | Path

Directory where downloaded raw data files are stored. Defaults to ".".

 '.'
processed_data_path str | Path

Base directory where the processed output data is saved. Defaults to ".".

 '.'
num_cores int

Number of CPU cores used for the IRBEM magnetic field computations (only used when use_level_3_orbit_data is False). Defaults to 4.

 4
cadence timedelta

Time binning cadence applied to all variables. Defaults to timedelta(minutes=5).

 timedelta(minutes=5)
save_strategy Literal['gfz', 'h5', 'netcdf']

The saving strategy used to write the processed data. Defaults to "gfz".

 'gfz'
data_standard Literal['gfz', 'prbem']

The data standard used when saving the processed data. Defaults to "gfz".

 'gfz'
use_level_3_orbit_data bool

If True, use Arase Level 3 orbit data (which already contains precomputed magnetic field quantities for mag_field); if False, use Level 2 orbit data and compute the magnetic field quantities via IRBEM. Defaults to True.

 True
Source code in el_paso/recipes/arase/process_arase_mepe.py 
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
def process_arase_mepe(
    start_time: datetime,
    end_time: datetime,
    mag_field: Literal["T89", "TS04", "OP77Q"],
    raw_data_path: str | Path = ".",
    processed_data_path: str | Path = ".",
    num_cores: int = 4,
    cadence: timedelta = timedelta(minutes=5),
    save_strategy: Literal["gfz", "h5", "netcdf"] = "gfz",
    data_standard: Literal["gfz", "prbem"] = "gfz",
    *,
    use_level_3_orbit_data: bool = True,
) -> None:
    """Process Arase MEP-e Level 3 electron flux data and save derived products.

    Downloads the corresponding Arase orbit data (Level 3 if `use_level_3_orbit_data` is True,
    otherwise Level 2) and the daily Arase MEP-e Level 3 pitch-angle resolved flux (FEDU) CDF
    files for the requested time range, then sorts energies into ascending order, time-bins the
    flux, position and pitch-angle variables onto a common cadence, folds the pitch-angle
    distribution, applies a lower flux threshold, and computes the equatorial pitch angle and
    magnetic-field-related quantities (Lm, MLT, R0) either from the Level 3 orbit data directly
    or via IRBEM using the SM position from Level 2 orbit data. The resulting variables are
    saved using the requested saving strategy.

    Args:
        start_time (datetime): Start of the time range to process.
        end_time (datetime): End of the time range to process.
        mag_field (Literal["T89", "TS04", "OP77Q"]): The magnetic field model used for the
                                                    magnetic-field-related output variables.
        raw_data_path (str | Path, optional): Directory where downloaded raw data files are
                                            stored. Defaults to ".".
        processed_data_path (str | Path, optional): Base directory where the processed output
                                                    data is saved. Defaults to ".".
        num_cores (int, optional): Number of CPU cores used for the IRBEM magnetic field
                                computations (only used when `use_level_3_orbit_data` is
                                False). Defaults to 4.
        cadence (timedelta, optional): Time binning cadence applied to all variables.
                                    Defaults to timedelta(minutes=5).
        save_strategy (Literal["gfz", "h5", "netcdf"], optional): The saving strategy used to
                                                                write the processed data.
                                                                Defaults to "gfz".
        data_standard (Literal["gfz", "prbem"], optional): The data standard used when saving
                                                            the processed data. Defaults to "gfz".
        use_level_3_orbit_data (bool, optional): If True, use Arase Level 3 orbit data (which
                                                already contains precomputed magnetic field
                                                quantities for `mag_field`); if False, use Level 2
                                                orbit data and compute the magnetic field
                                                quantities via IRBEM. Defaults to True.
    """
    logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
    logging.getLogger().setLevel(logging.INFO)

    raw_data_path = Path(raw_data_path)
    processed_data_path = Path(processed_data_path)

    if use_level_3_orbit_data:
        orb_variables = get_arase_orbit_level_3_variables(start_time, end_time, mag_field, raw_data_path=raw_data_path)
    else:
        orb_variables = get_arase_orbit_level_2_variables(start_time, end_time, raw_data_path=raw_data_path)

    file_name_stem = "erg_mepe_l3_pa_YYYYMMDD_.{6}.cdf"
    url = "https://ergsc.isee.nagoya-u.ac.jp/data/ergsc/satellite/erg/mepe/l3/pa/YYYY/MM/"

    ep.download(
        start_time,
        end_time,
        save_path=raw_data_path,
        download_url=url,
        file_name_stem=file_name_stem,
        file_cadence="daily",
        method="request",
        skip_existing=True,
    )

    extraction_infos = [
        ep.ExtractionInfo(
            result_key="Epoch",
            name_or_column="epoch",
            unit=ep.units.tt2000,
        ),
        ep.ExtractionInfo(
            result_key="Energy",
            name_or_column="FEDU_Energy",
            unit=u.keV,
            is_time_dependent=False,
        ),
        ep.ExtractionInfo(
            result_key="Pitch_angle",
            name_or_column="FEDU_Alpha",
            unit=u.deg,
            is_time_dependent=False,
        ),
        ep.ExtractionInfo(
            result_key="FEDU",
            name_or_column="FEDU",
            unit=(u.cm**2 * u.s * u.sr * u.keV) ** (-1),
        ),
    ]

    mepe_variables = ep.extract_variables_from_files(
        start_time,
        end_time,
        "daily",
        data_path=raw_data_path,
        file_name_stem=file_name_stem,
        extraction_infos=extraction_infos,
    )

    mepe_variables["FEDU"].truncate(mepe_variables["Epoch"], start_time, end_time)
    mepe_variables["Epoch"].truncate(mepe_variables["Epoch"], start_time, end_time)

    # sort energies into ascending order
    idx_sorted = np.argsort(mepe_variables["Energy"].get_data())
    mepe_variables["Energy"].set_data(mepe_variables["Energy"].get_data()[idx_sorted[:-1]], "same")
    mepe_variables["FEDU"].set_data(mepe_variables["FEDU"].get_data()[:, idx_sorted[:-1], :], "same")

    time_bin_methods = {
        "xGEO": ep.TimeBinMethod.NanMean,
        "Energy": ep.TimeBinMethod.Repeat,
        "FEDU": ep.TimeBinMethod.NanMedian,
        "Pitch_angle": ep.TimeBinMethod.Repeat,
    }

    binned_time_variable = ep.processing.bin_by_time(
        mepe_variables["Epoch"],
        variables=mepe_variables,
        time_bin_method_dict=time_bin_methods,
        time_binning_cadence=cadence,
        start_time=start_time,
        end_time=end_time,
    )

    ep.processing.fold_pitch_angles_and_flux(mepe_variables["FEDU"], mepe_variables["Pitch_angle"])

    mepe_variables["FEDU"].apply_thresholds_on_data(lower_threshold=1e-21)

    if use_level_3_orbit_data:
        time_bin_methods = {
            "B_local": ep.TimeBinMethod.NanMedian,
            "B_eq": ep.TimeBinMethod.NanMedian,
            "Lm": ep.TimeBinMethod.NanMean,
            "Lstar": ep.TimeBinMethod.NanMean,
            "MLT": ep.TimeBinMethod.NanMean,
            "R0": ep.TimeBinMethod.NanMean,
        }

        binned_time_variable = ep.processing.bin_by_time(
            orb_variables["Epoch"],
            variables=orb_variables,
            time_bin_method_dict=time_bin_methods,
            time_binning_cadence=cadence,
            start_time=start_time,
            end_time=end_time,
        )

        pa_local = mepe_variables["Pitch_angle"].get_data(u.radian)
        pa_eq = np.asin(
            np.sin(pa_local)
            * np.sqrt(orb_variables["B_eq"].get_data(u.nT) / orb_variables["B_local"].get_data(u.nT))[:, np.newaxis]
        )
        mepe_variables["Pa_eq"] = ep.Variable(data=pa_eq, original_unit=u.radian)

    else:
        time_bin_methods = {
            "pos_sm": ep.TimeBinMethod.NanMean,
        }

        binned_time_variable = ep.processing.bin_by_time(
            orb_variables["Epoch"],
            variables=orb_variables,
            time_bin_method_dict=time_bin_methods,
            time_binning_cadence=cadence,
            start_time=start_time,
            end_time=end_time,
        )

        datetimes = [
            datetime.fromtimestamp(t, tz=timezone.utc) for t in binned_time_variable.get_data(ep.units.posixtime)
        ]

        geo_data = Coords().transform(
            time=datetimes,
            pos=orb_variables["pos_sm"].get_data().astype(np.float64),
            sysaxes_in=ep.IRBEM_SYSAXIS_SM,
            sysaxes_out=ep.IRBEM_SYSAXIS_GEO,
        )
        pos_geo_var = ep.Variable(data=geo_data, original_unit=ep.units.RE)

        irbem_options = ep.processing.magnetic_field_utils.IrbemOptions()

        variables_to_compute: ep.processing.VariableRequest = [
            ("B_Calc", mag_field),
            ("MLT", mag_field),
            ("B_Eq", mag_field),
            ("R_Eq", mag_field),
            ("Alpha_Eq", mag_field),
            ("L_m", mag_field),
        ]

        magnetic_field_variables = ep.processing.compute_magnetic_field_variables(
            time_var=binned_time_variable,
            xgeo_var=pos_geo_var,
            variables_to_compute=variables_to_compute,
            irbem_options=irbem_options,
            num_cores=num_cores,
            pa_local_var=mepe_variables["Pitch_angle"],
        )

        orb_variables["R0"] = magnetic_field_variables["R_Eq_" + mag_field]
        orb_variables["MLT"] = magnetic_field_variables["MLT_" + mag_field]
        mepe_variables["Pa_eq"] = magnetic_field_variables["Alpha_Eq_" + mag_field]
        orb_variables["Lm"] = magnetic_field_variables["L_m_" + mag_field]

    match mag_field:
        case "T89":
            mag_field_save = "T89"
        case "TS04":
            mag_field_save = "T04s"
        case "OP77Q":
            mag_field_save = "OP77"

    data_standard_instance = (
        ep.data_standards.GFZStandard() if data_standard == "gfz" else ep.data_standards.PRBEMStandard()
    )

    variables_to_save: dict[ep.typing.InternalName, ep.Variable] = {
        "Epoch": binned_time_variable,
        "FEDU": mepe_variables["FEDU"],
        "Energy_FEDU": mepe_variables["Energy"],
        "Alpha": mepe_variables["Pitch_angle"],
        "Alpha_Eq": mepe_variables["Pa_eq"],
        "R_Eq": orb_variables["R0"],
        "MLT": orb_variables["MLT"],
        "L_m": orb_variables["Lm"],
    }

    match save_strategy:
        case "gfz":
            saving_strategy = ep.saving_strategies.GFZStrategy(
                processed_data_path,
                "ARASE",
                "arase",
                "mepe",
                mag_field_save,
                data_standard_instance,
            )

        case "h5":
            saving_strategy = ep.saving_strategies.MonthlyRBStrategy(
                processed_data_path,
                "Arase",
                "arase",
                "mepe",
                mag_field=mag_field,
                file_format="h5",
                data_standard=data_standard_instance,
            )

        case "netcdf":
            saving_strategy = ep.saving_strategies.MonthlyRBStrategy(
                processed_data_path,
                "Arase",
                "arase",
                "mepe",
                mag_field=mag_field,
                file_format="nc",
                data_standard=data_standard_instance,
            )

    ep.save(variables_to_save, saving_strategy, start_time, end_time, binned_time_variable)

el_paso.recipes.arase.process_arase_pwe_densities.process_arase_pwe_density 

process_arase_pwe_density

Process Arase PWE/HFA electron density data and save the mapped equatorial density.

Downloads the Arase Level 2 orbit data and the daily Arase PWE/HFA Level 3 1-minute electron density CDF files for the requested time range, time-bins the density and the SM position onto a common cadence, applies a lower density threshold, converts the position to GEO coordinates, computes the magnetic-field-related quantities (MLT, equatorial radial distance and equatorial position) via IRBEM for the given mag_field, maps the local density to the magnetic equator, and saves the resulting variables using a DensityNetCDFStrategy.

Parameters:

Name Type Description Default
start_time datetime

Start of the time range to process.

 required
end_time datetime

End of the time range to process.

 required
mag_field Literal['T89', 'TS04', 'OP77Q']

The magnetic field model used for the magnetic-field-related output variables and the equatorial density mapping.

 required
raw_data_path str | Path

Directory where downloaded raw data files are stored. Defaults to ".".

 '.'
processed_data_path str | Path

Base directory where the processed output data is saved. Defaults to ".".

 '.'
num_cores int

Number of CPU cores used for the IRBEM magnetic field computations. Defaults to 4.

 4
cadence timedelta

Time binning cadence applied to all variables. Defaults to timedelta(minutes=5).

 timedelta(minutes=5)
Source code in el_paso/recipes/arase/process_arase_pwe_densities.py 
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
def process_arase_pwe_density(
    start_time: datetime,
    end_time: datetime,
    mag_field: Literal["T89", "TS04", "OP77Q"],
    raw_data_path: str | Path = ".",
    processed_data_path: str | Path = ".",
    num_cores: int = 4,
    cadence: timedelta = timedelta(minutes=5),
) -> None:
    """Process Arase PWE/HFA electron density data and save the mapped equatorial density.

    Downloads the Arase Level 2 orbit data and the daily Arase PWE/HFA Level 3 1-minute electron
    density CDF files for the requested time range, time-bins the density and the SM position
    onto a common cadence, applies a lower density threshold, converts the position to GEO
    coordinates, computes the magnetic-field-related quantities (MLT, equatorial radial distance
    and equatorial position) via IRBEM for the given `mag_field`, maps the local density to the
    magnetic equator, and saves the resulting variables using a `DensityNetCDFStrategy`.

    Args:
        start_time (datetime): Start of the time range to process.
        end_time (datetime): End of the time range to process.
        mag_field (Literal["T89", "TS04", "OP77Q"]): The magnetic field model used for the
                                                    magnetic-field-related output variables and
                                                    the equatorial density mapping.
        raw_data_path (str | Path, optional): Directory where downloaded raw data files are
                                            stored. Defaults to ".".
        processed_data_path (str | Path, optional): Base directory where the processed output
                                                    data is saved. Defaults to ".".
        num_cores (int, optional): Number of CPU cores used for the IRBEM magnetic field
                                computations. Defaults to 4.
        cadence (timedelta, optional): Time binning cadence applied to all variables.
                                    Defaults to timedelta(minutes=5).
    """
    logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
    logging.getLogger().setLevel(logging.INFO)

    raw_data_path = Path(raw_data_path)
    processed_data_path = Path(processed_data_path)

    orb_variables = get_arase_orbit_level_2_variables(start_time, end_time, raw_data_path=raw_data_path)

    file_name_stem = "erg_pwe_hfa_l3_1min_YYYYMMDD_.{6}.cdf"
    url = "https://ergsc.isee.nagoya-u.ac.jp/data/ergsc/satellite/erg/pwe/hfa/l3/YYYY/MM/"

    ep.download(
        start_time,
        end_time,
        save_path=raw_data_path,
        download_url=url,
        file_name_stem=file_name_stem,
        file_cadence="daily",
        method="request",
        skip_existing=True,
    )

    extraction_infos = [
        ep.ExtractionInfo(
            result_key="Epoch",
            name_or_column="Epoch",
            unit=ep.units.tt2000,
        ),
        ep.ExtractionInfo(
            result_key="Density",
            name_or_column="ne_mgf",
            unit=u.cm ** (-3),
        ),
    ]

    pwe_variables = ep.extract_variables_from_files(
        start_time,
        end_time,
        "daily",
        data_path=raw_data_path,
        file_name_stem=file_name_stem,
        extraction_infos=extraction_infos,
    )

    time_bin_methods = {
        "Density": ep.TimeBinMethod.NanMedian,
    }

    binned_time_variable = ep.processing.bin_by_time(
        pwe_variables["Epoch"],
        variables=pwe_variables,
        time_bin_method_dict=time_bin_methods,
        time_binning_cadence=cadence,
        start_time=start_time,
        end_time=end_time,
    )

    pwe_variables["Density"].apply_thresholds_on_data(lower_threshold=1e-21)

    time_bin_methods = {
        "pos_sm": ep.TimeBinMethod.NanMean,
    }

    binned_time_variable = ep.processing.bin_by_time(
        orb_variables["Epoch"],
        variables=orb_variables,
        time_bin_method_dict=time_bin_methods,
        time_binning_cadence=cadence,
        start_time=start_time,
        end_time=end_time,
    )

    datetimes = [datetime.fromtimestamp(t, tz=timezone.utc) for t in binned_time_variable.get_data(ep.units.posixtime)]

    geo_data = Coords().transform(
        time=datetimes,
        pos=orb_variables["pos_sm"].get_data().astype(np.float64),
        sysaxes_in=ep.IRBEM_SYSAXIS_SM,
        sysaxes_out=ep.IRBEM_SYSAXIS_GEO,
    )
    pos_geo_var = ep.Variable(data=geo_data, original_unit=ep.units.RE)

    irbem_options = ep.processing.magnetic_field_utils.IrbemOptions()

    variables_to_compute: ep.processing.VariableRequest = [
        ("MLT", mag_field),
        ("R_Eq", mag_field),
        ("xGEO_Eq", mag_field),
    ]

    magnetic_field_variables = ep.processing.compute_magnetic_field_variables(
        time_var=binned_time_variable,
        xgeo_var=pos_geo_var,
        variables_to_compute=variables_to_compute,
        irbem_options=irbem_options,
        num_cores=num_cores,
    )

    pwe_variables["Density_mapped_" + mag_field] = ep.processing.compute_equatorial_plasmaspheric_density(
        pwe_variables["Density"], pos_geo_var, magnetic_field_variables["xGEO_eq_" + mag_field], method="Denton_average"
    )

    saving_strategy = ep.saving_strategies.DensityNetCDFStrategy(
        base_data_path=processed_data_path,
        mission="Arase",
        satellite="Other",
        instrument="PWE",
        mag_field=mag_field,
    )

    variables_to_save = {
        "time": binned_time_variable,
        "density_local": pwe_variables["Density"],
        "density_eq": pwe_variables["Density_mapped_" + mag_field],
        "MLT": magnetic_field_variables["MLT_" + mag_field],
        "R_eq": magnetic_field_variables["R_Eq_" + mag_field],
        "xGEO": pos_geo_var,
        "xGEO_eq": magnetic_field_variables["xGEO_Eq_" + mag_field],
    }

    ep.save(variables_to_save, saving_strategy, start_time, end_time, binned_time_variable)  # ty:ignore[invalid-argument-type]

el_paso.recipes.arase.process_arase_xep_realtime.process_arase_xep_real_time 

process_arase_xep_real_time

Process Arase XEP real-time electron flux data and save derived products.

Downloads (unless download is False) and extracts the daily Arase real-time XEP omnidirectional flux (FEDO) text files and the real-time orbit position text files for the requested time range, converts the orbit position from SM to GEO coordinates, time-bins the flux and position variables onto a 5-minute cadence, computes magnetic-field-related quantities (B_Calc, B_Eq, MLT, R_Eq, Alpha_Eq, L_star, L_m, InvMu, InvK) for the T89 model via IRBEM, constructs a pitch-angle distribution (FEDU) from the omnidirectional flux, applies a lower flux threshold, computes the electron phase space density, and saves the resulting variables using the requested saving strategy/strategies.

Parameters:

Name Type Description Default
processed_data_path str | Path

Base directory where the processed output data is saved.

 required
download_data_dir str | Path

Base directory where downloaded raw data files are stored.

 required
start_time datetime

Start of the time range to process.

 required
end_time datetime

End of the time range to process.

 required
erg_user str | None

Username for the ERG data server. If None, it is read from the ERG_USER environment variable. Defaults to None.

 None
erg_password str | None

Password for the ERG data server. If None, it is read from the ERG_PASSWORD environment variable. Defaults to None.

 None
num_cores int

Number of CPU cores used for the IRBEM magnetic field computations. Defaults to 32.

 32
save_strategy Literal['gfz', 'netcdf', 'both']

Which saving strategy/strategies to use for writing the processed data. Defaults to "netcdf".

 'netcdf'
download bool

Whether to download the raw data files before processing. Defaults to True.

 True
skip_existing bool

Whether to skip downloading files that already exist locally. Defaults to True.

 True

Raises:

Type Description
ValueError

If erg_user is not provided and the ERG_USER environment variable is not set, or if erg_password is not provided and the ERG_PASSWORD environment variable is not set.
Source code in el_paso/recipes/arase/process_arase_xep_realtime.py 
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
@timed_function("process_arase_xep_real_time")
def process_arase_xep_real_time(
    processed_data_path: str | Path,
    download_data_dir: str | Path,
    start_time: datetime,
    end_time: datetime,
    erg_user: str | None = None,
    erg_password: str | None = None,
    num_cores: int = 32,
    save_strategy: Literal["gfz", "netcdf", "both"] = "netcdf",
    *,
    download: bool = True,
    skip_existing: bool = True,
) -> None:
    """Process Arase XEP real-time electron flux data and save derived products.

    Downloads (unless `download` is False) and extracts the daily Arase real-time XEP
    omnidirectional flux (FEDO) text files and the real-time orbit position text files for the
    requested time range, converts the orbit position from SM to GEO coordinates, time-bins the
    flux and position variables onto a 5-minute cadence, computes magnetic-field-related
    quantities (B_Calc, B_Eq, MLT, R_Eq, Alpha_Eq, L_star, L_m, InvMu, InvK) for the T89 model via
    IRBEM, constructs a pitch-angle distribution (FEDU) from the omnidirectional flux, applies a
    lower flux threshold, computes the electron phase space density, and saves the resulting
    variables using the requested saving strategy/strategies.

    Args:
        processed_data_path (str | Path): Base directory where the processed output data is saved.
        download_data_dir (str | Path): Base directory where downloaded raw data files are stored.
        start_time (datetime): Start of the time range to process.
        end_time (datetime): End of the time range to process.
        erg_user (str | None, optional): Username for the ERG data server. If None, it is read
                                        from the ``ERG_USER`` environment variable.
                                        Defaults to None.
        erg_password (str | None, optional): Password for the ERG data server. If None, it is
                                            read from the ``ERG_PASSWORD`` environment variable.
                                            Defaults to None.
        num_cores (int, optional): Number of CPU cores used for the IRBEM magnetic field
                                computations. Defaults to 32.
        save_strategy (Literal["gfz", "netcdf", "both"], optional): Which saving strategy/strategies
                                                                    to use for writing the processed
                                                                    data. Defaults to "netcdf".
        download (bool, optional): Whether to download the raw data files before processing.
                                Defaults to True.
        skip_existing (bool, optional): Whether to skip downloading files that already exist
                                        locally. Defaults to True.

    Raises:
        ValueError: If `erg_user` is not provided and the ``ERG_USER`` environment variable is
                not set, or if `erg_password` is not provided and the ``ERG_PASSWORD``
                environment variable is not set.
    """
    logging.basicConfig(stream=sys.stdout, level=logging.INFO)

    if erg_user is None:
        erg_user = os.environ.get("ERG_USER")
    if erg_password is None:
        erg_password = os.environ.get("ERG_PASSWORD")

    if erg_user is None:
        msg = "ERG_USER not found! Either load it from environment variables or pass it as an argument."
        raise ValueError(msg)

    if erg_password is None:
        msg = "ERG_PASSWORD not found! Either load it from environment variables or pass it as an argument."
        raise ValueError(msg)

    xep_variables = _get_xep_variables(
        download_data_dir, start_time, end_time, erg_user, erg_password, download=download, skip_existing=skip_existing
    )
    orb_variables = _get_orb_variables(
        download_data_dir,
        start_time,
        end_time,
        erg_user,
        erg_password,
        download=download,
        skip_existing=skip_existing,
    )

    time_bin_methods_xep = {
        "FEDO": ep.TimeBinMethod.NanMedian,
        "Energy_FEDO": ep.TimeBinMethod.Repeat,
        "PA_local_FEDU": ep.TimeBinMethod.Repeat,
    }
    binned_time_var = ep.processing.bin_by_time(
        xep_variables["Epoch"],
        xep_variables,
        time_bin_methods_xep,
        time_binning_cadence=timedelta(minutes=5),
        start_time=start_time,
        end_time=end_time,
    )

    time_bin_methods_orb = {
        "xGEO": ep.TimeBinMethod.NanMedian,
    }
    _ = ep.processing.bin_by_time(
        orb_variables["Epoch"],
        orb_variables,
        time_bin_methods_orb,
        time_binning_cadence=timedelta(minutes=5),
        start_time=start_time,
        end_time=end_time,
    )

    variables_combined = xep_variables | orb_variables

    variables_to_compute: ep.processing.VariableRequest = [
        ("B_Calc", "T89"),
        ("B_Eq", "T89"),
        ("MLT", "T89"),
        ("R_Eq", "T89"),
        ("Alpha_Eq", "T89"),
        ("L_star", "T89"),
        ("L_m", "T89"),
        ("InvMu", "T89"),
        ("InvK", "T89"),
    ]

    magnetic_field_variables = ep.processing.compute_magnetic_field_variables(
        time_var=binned_time_var,
        xgeo_var=variables_combined["xGEO"],
        energy_var=variables_combined["Energy_FEDO"],
        pa_local_var=variables_combined["PA_local_FEDU"],
        particle_species="electron",
        variables_to_compute=variables_to_compute,
        irbem_options=ep.processing.magnetic_field_utils.IrbemOptions(),
        num_cores=num_cores,
    )

    variables_combined |= magnetic_field_variables

    FEDU_var = ep.processing.construct_pitch_angle_distribution(
        variables_combined["FEDO"], variables_combined["PA_local_FEDU"], magnetic_field_variables["Alpha_Eq_T89"]
    )
    FEDU_var.apply_thresholds_on_data(lower_threshold=0)

    psd_var = ep.processing.compute_phase_space_density(
        FEDU_var, variables_combined["Energy_FEDO"], particle_species="electron"
    )

    variables_to_save: dict[ep.typing.InternalName, ep.Variable] = {
        "Epoch": binned_time_var,
        "FEDU": FEDU_var,
        "Energy_FEDU": variables_combined["Energy_FEDO"],
        "Alpha": variables_combined["PA_local_FEDU"],
        "Alpha_Eq": magnetic_field_variables["Alpha_Eq_T89"],
        "R_Eq": magnetic_field_variables["R_Eq_T89"],
        "MLT": magnetic_field_variables["MLT_T89"],
        "L_star": magnetic_field_variables["L_star_T89"],
        "B_Calc": magnetic_field_variables["B_Calc_T89"],
        "B_Eq": magnetic_field_variables["B_Eq_T89"],
        "PSD": psd_var,
        "InvMu": magnetic_field_variables["InvMu_T89"],
        "InvK": magnetic_field_variables["InvK_T89"],
        "Position": variables_combined["xGEO"],
    }

    if save_strategy in ("gfz", "both"):
        saving_strategy = ep.saving_strategies.GFZStrategy(
            processed_data_path,
            mission="Arase",
            satellite="Arase",
            instrument="XEP",
            mag_field="T89",
        )

        ep.save(
            variables_to_save,
            saving_strategy,
            start_time,
            end_time,
            time_var=binned_time_var,
            append=True,
        )

    if save_strategy in ("netcdf", "both"):
        saving_strategy = ep.saving_strategies.MonthlyRBStrategy(
            base_data_path=Path(processed_data_path),
            mission="Arase",
            satellite="arase",
            instrument="xep",
            mag_field="T89",
            data_standard=ep.data_standards.GFZStandard(),
        )

        ep.save(
            variables_to_save,
            saving_strategy,
            start_time,
            end_time,
            time_var=binned_time_var,
            append=True,
        )