From d6f0db9586e4194b1c6b138cd22cb7fd95d8b479 Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Wed, 8 Jul 2026 17:48:41 +1000 Subject: [PATCH 1/2] Add FXMacroData market data source --- alphapy/data.py | 60 ++++++++++++++++++++++++++++++++++ tests/test_fxmacrodata_data.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 tests/test_fxmacrodata_data.py diff --git a/alphapy/data.py b/alphapy/data.py index b2f4bcc..9e0517a 100644 --- a/alphapy/data.py +++ b/alphapy/data.py @@ -76,6 +76,8 @@ logger = logging.getLogger(__name__) +FXMACRODATA_API_ROOT = 'https://fxmacrodata.com/api/v1' + # # Function get_data @@ -724,11 +726,69 @@ def get_yahoo_data(schema, subschema, symbol, intraday_data, data_fractal, return df +# +# Function get_fxmacrodata_data +# + +def get_fxmacrodata_data(schema, subschema, symbol, intraday_data, data_fractal, + from_date, to_date, lookback_period): + r"""Get daily FX reference rates from FXMacroData. + + FXMacroData returns one official reference value per currency pair and + date. The value is copied into open, high, low, and close with zero volume + so MarketFlow can consume it through the normal OHLCV path. + + """ + + df = pd.DataFrame() + if intraday_data: + logger.info("FXMacroData supports daily reference rates, not intraday bars") + return df + + pair = symbol.upper().replace('/', '').replace('-', '').replace('_', '') + if len(pair) != 6: + logger.error("FXMacroData symbol must be formatted like EURUSD or EUR/USD") + return df + + base = pair[:3] + quote = pair[3:] + url = SSEP.join([FXMACRODATA_API_ROOT.rstrip('/'), 'forex', base, quote]) + params = { + 'start_date': from_date, + 'end_date': to_date, + 'limit': 5000, + } + api_key = os.environ.get('FXMACRODATA_API_KEY') + if api_key: + params['api_key'] = api_key + + try: + response = requests.get(url, params=params) + response.raise_for_status() + rows = response.json().get('data', []) + except Exception: + logger.info("Could not retrieve %s data with FXMacroData", symbol.upper()) + return df + + records = [] + for row in rows: + value = float(row['val']) + records.append((row['date'], value, value, value, value, 0.0)) + + if records: + df = pd.DataFrame.from_records( + records, + columns=['date', 'open', 'high', 'low', 'close', 'volume']) + + return df + + # # Data Dispatch Tables # data_dispatch_table = {'google' : get_google_data, + 'fxmacrodata' : get_fxmacrodata_data, 'iex' : get_iex_data, 'pandas' : get_pandas_data, 'quandl' : get_quandl_data, diff --git a/tests/test_fxmacrodata_data.py b/tests/test_fxmacrodata_data.py new file mode 100644 index 0000000..a64ee60 --- /dev/null +++ b/tests/test_fxmacrodata_data.py @@ -0,0 +1,51 @@ +import unittest + +from alphapy import data + + +class TestFXMacroDataData(unittest.TestCase): + + def test_get_fxmacrodata_data(self): + class MockResponse: + def raise_for_status(self): + pass + + def json(self): + return { + 'data': [ + {'date': '2026-01-02', 'val': 1.2}, + {'date': '2026-01-01', 'val': 1.1}, + ] + } + + calls = {} + + def mock_get(url, params): + calls['url'] = url + calls['params'] = params + return MockResponse() + + original_get = data.requests.get + try: + data.requests.get = mock_get + df = data.get_fxmacrodata_data( + 'fxmacrodata', + '', + 'EUR/USD', + False, + '1D', + '2026-01-01', + '2026-01-02', + 2, + ) + finally: + data.requests.get = original_get + + self.assertEqual(calls['url'], 'https://fxmacrodata.com/api/v1/forex/EUR/USD') + self.assertEqual(calls['params']['start_date'], '2026-01-01') + self.assertEqual(list(df.columns), ['date', 'open', 'high', 'low', 'close', 'volume']) + self.assertEqual(list(df['close']), [1.2, 1.1]) + + +if __name__ == '__main__': + unittest.main() From 4acd900e2d73a3656112e54db6661e374e952b11 Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Sun, 16 Aug 2026 18:03:44 +1000 Subject: [PATCH 2/2] Harden MarketFlow FX data loading --- alphapy/data.py | 61 +----------------- alphapy/fxmacrodata.py | 106 ++++++++++++++++++++++++++++++++ docs/user_guide/market_flow.rst | 13 ++++ tests/test_fxmacrodata_data.py | 75 +++++++++++++++++++--- 4 files changed, 185 insertions(+), 70 deletions(-) create mode 100644 alphapy/fxmacrodata.py diff --git a/alphapy/data.py b/alphapy/data.py index 9e0517a..629cf87 100644 --- a/alphapy/data.py +++ b/alphapy/data.py @@ -29,6 +29,7 @@ from alphapy.frame import Frame from alphapy.frame import frame_name from alphapy.frame import read_frame +from alphapy.fxmacrodata import get_fxmacrodata_data from alphapy.globals import ModelType from alphapy.globals import Partition, datasets from alphapy.globals import PSEP, SSEP, USEP @@ -76,9 +77,6 @@ logger = logging.getLogger(__name__) -FXMACRODATA_API_ROOT = 'https://fxmacrodata.com/api/v1' - - # # Function get_data # @@ -726,63 +724,6 @@ def get_yahoo_data(schema, subschema, symbol, intraday_data, data_fractal, return df -# -# Function get_fxmacrodata_data -# - -def get_fxmacrodata_data(schema, subschema, symbol, intraday_data, data_fractal, - from_date, to_date, lookback_period): - r"""Get daily FX reference rates from FXMacroData. - - FXMacroData returns one official reference value per currency pair and - date. The value is copied into open, high, low, and close with zero volume - so MarketFlow can consume it through the normal OHLCV path. - - """ - - df = pd.DataFrame() - if intraday_data: - logger.info("FXMacroData supports daily reference rates, not intraday bars") - return df - - pair = symbol.upper().replace('/', '').replace('-', '').replace('_', '') - if len(pair) != 6: - logger.error("FXMacroData symbol must be formatted like EURUSD or EUR/USD") - return df - - base = pair[:3] - quote = pair[3:] - url = SSEP.join([FXMACRODATA_API_ROOT.rstrip('/'), 'forex', base, quote]) - params = { - 'start_date': from_date, - 'end_date': to_date, - 'limit': 5000, - } - api_key = os.environ.get('FXMACRODATA_API_KEY') - if api_key: - params['api_key'] = api_key - - try: - response = requests.get(url, params=params) - response.raise_for_status() - rows = response.json().get('data', []) - except Exception: - logger.info("Could not retrieve %s data with FXMacroData", symbol.upper()) - return df - - records = [] - for row in rows: - value = float(row['val']) - records.append((row['date'], value, value, value, value, 0.0)) - - if records: - df = pd.DataFrame.from_records( - records, - columns=['date', 'open', 'high', 'low', 'close', 'volume']) - - return df - - # # Data Dispatch Tables # diff --git a/alphapy/fxmacrodata.py b/alphapy/fxmacrodata.py new file mode 100644 index 0000000..ac48c8a --- /dev/null +++ b/alphapy/fxmacrodata.py @@ -0,0 +1,106 @@ +################################################################################ +# +# Package : AlphaPy +# Module : fxmacrodata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +################################################################################ + +"""FXMacroData daily FX reference-rate adapter for MarketFlow.""" + +import logging +import os + +import pandas as pd +import requests + + +logger = logging.getLogger(__name__) + +FXMACRODATA_API_ROOT = 'https://api.fxmacrodata.com/v1' +FXMACRODATA_PAGE_SIZE = 100 + + +def get_fxmacrodata_data(schema, subschema, symbol, intraday_data, data_fractal, + from_date, to_date, lookback_period): + r"""Get daily FX reference rates from FXMacroData. + + The parameters match AlphaPy's market-data dispatch contract. Native + reference-observation OHLC is preserved when present; otherwise the daily + reference value is copied into OHLC and volume is set to zero. + + """ + + df = pd.DataFrame() + if intraday_data: + logger.info("FXMacroData supports daily reference rates, not intraday bars") + return df + + pair = symbol.upper().replace('/', '').replace('-', '').replace('_', '') + if len(pair) != 6 or not pair.isalpha() or not pair.isascii(): + logger.error("FXMacroData symbol must be formatted like EURUSD or EUR/USD") + return df + + base = pair[:3] + quote = pair[3:] + url = '/'.join([FXMACRODATA_API_ROOT.rstrip('/'), 'forex', base, quote]) + base_params = {'start_date': from_date, 'end_date': to_date} + api_key = (os.environ.get('FXMACRODATA_API_KEY') or + os.environ.get('FXMD_API_KEY')) + if api_key: + base_params['api_key'] = api_key + + rows = [] + offset = 0 + try: + while True: + params = dict(base_params) + params.update({'limit': FXMACRODATA_PAGE_SIZE, 'offset': offset}) + response = requests.get(url, params=params, timeout=30) + if not response.ok: + logger.info("FXMacroData returned HTTP %s for %s", + response.status_code, symbol.upper()) + return df + payload = response.json() + page = payload.get('data', []) if isinstance(payload, dict) else [] + if not isinstance(page, list): + logger.info("FXMacroData returned invalid data for %s", symbol.upper()) + return df + rows.extend(row for row in page if isinstance(row, dict)) + if len(page) < FXMACRODATA_PAGE_SIZE: + break + offset += FXMACRODATA_PAGE_SIZE + except (requests.RequestException, ValueError, TypeError): + logger.info("Could not retrieve %s data with FXMacroData", symbol.upper()) + return df + + records = [] + for row in rows: + try: + value = float(row['val']) + record = ( + row['date'], + float(row.get('open', value)), + float(row.get('high', value)), + float(row.get('low', value)), + float(row.get('close', value)), + 0.0, + ) + except (KeyError, TypeError, ValueError): + continue + records.append(record) + + if records: + df = pd.DataFrame.from_records( + records, + columns=['date', 'open', 'high', 'low', 'close', 'volume']) + df.drop_duplicates(subset=['date'], keep='first', inplace=True) + df.sort_values('date', inplace=True) + df.reset_index(drop=True, inplace=True) + + return df diff --git a/docs/user_guide/market_flow.rst b/docs/user_guide/market_flow.rst index 9a70eeb..ce6eab2 100644 --- a/docs/user_guide/market_flow.rst +++ b/docs/user_guide/market_flow.rst @@ -41,6 +41,19 @@ that are shortened. we recommend that you save the data on an ongoing basis for a a larger backtesting window. +FXMacroData daily FX reference rates are available through the +``fxmacrodata`` schema. Use six-letter currency-pair subjects such as +``EURUSD`` (``EUR/USD`` is also accepted). FX history normally requires an +API key, read from ``FXMACRODATA_API_KEY`` or ``FXMD_API_KEY`` in the +environment. Returned official reference observations are mapped to +the standard OHLCV frame. When the API provides reference-observation OHLC, +those values are preserved; otherwise the daily reference value is used for +all four price fields and volume is zero. Intraday retrieval is not supported. + +This MarketFlow adapter supports FX rates only. FXMacroData catalogue, +macroeconomic history, release calendar, forecasts, COT, commodities, +sessions, news, and seasonality surfaces are not exposed by AlphaPy. + Domain Configuration -------------------- diff --git a/tests/test_fxmacrodata_data.py b/tests/test_fxmacrodata_data.py index a64ee60..242838f 100644 --- a/tests/test_fxmacrodata_data.py +++ b/tests/test_fxmacrodata_data.py @@ -1,14 +1,14 @@ import unittest -from alphapy import data +from alphapy import fxmacrodata class TestFXMacroDataData(unittest.TestCase): def test_get_fxmacrodata_data(self): class MockResponse: - def raise_for_status(self): - pass + ok = True + status_code = 200 def json(self): return { @@ -20,15 +20,16 @@ def json(self): calls = {} - def mock_get(url, params): + def mock_get(url, params, timeout): calls['url'] = url calls['params'] = params + calls['timeout'] = timeout return MockResponse() - original_get = data.requests.get + original_get = fxmacrodata.requests.get try: - data.requests.get = mock_get - df = data.get_fxmacrodata_data( + fxmacrodata.requests.get = mock_get + df = fxmacrodata.get_fxmacrodata_data( 'fxmacrodata', '', 'EUR/USD', @@ -39,12 +40,66 @@ def mock_get(url, params): 2, ) finally: - data.requests.get = original_get + fxmacrodata.requests.get = original_get - self.assertEqual(calls['url'], 'https://fxmacrodata.com/api/v1/forex/EUR/USD') + self.assertEqual(calls['url'], 'https://api.fxmacrodata.com/v1/forex/EUR/USD') self.assertEqual(calls['params']['start_date'], '2026-01-01') + self.assertEqual(calls['params']['limit'], 100) + self.assertEqual(calls['timeout'], 30) self.assertEqual(list(df.columns), ['date', 'open', 'high', 'low', 'close', 'volume']) - self.assertEqual(list(df['close']), [1.2, 1.1]) + self.assertEqual(list(df['close']), [1.1, 1.2]) + + def test_get_fxmacrodata_data_preserves_ohlc_and_paginates(self): + calls = [] + + class MockResponse: + ok = True + status_code = 200 + + def __init__(self, rows): + self.rows = rows + + def json(self): + return {'data': self.rows} + + first_page = [ + {'date': '2026-01-02', 'val': 1.2, + 'open': 1.1, 'high': 1.3, 'low': 1.0, 'close': 1.25} + ] * 100 + second_page = [{'date': '2026-01-01', 'val': '1.05'}] + + def mock_get(url, params, timeout): + calls.append(dict(params)) + rows = first_page if params['offset'] == 0 else second_page + return MockResponse(rows) + + original_get = fxmacrodata.requests.get + try: + fxmacrodata.requests.get = mock_get + df = fxmacrodata.get_fxmacrodata_data( + 'fxmacrodata', '', 'EURUSD', False, '1D', + '2026-01-01', '2026-01-02', 2) + finally: + fxmacrodata.requests.get = original_get + + self.assertEqual([call['offset'] for call in calls], [0, 100]) + self.assertEqual(list(df['date']), ['2026-01-01', '2026-01-02']) + self.assertEqual(df.iloc[1]['open'], 1.1) + self.assertEqual(df.iloc[1]['high'], 1.3) + self.assertEqual(df.iloc[1]['low'], 1.0) + self.assertEqual(df.iloc[1]['close'], 1.25) + + def test_get_fxmacrodata_data_rejects_invalid_pair_without_request(self): + original_get = fxmacrodata.requests.get + try: + fxmacrodata.requests.get = lambda *args, **kwargs: self.fail('unexpected request') + df = fxmacrodata.get_fxmacrodata_data( + 'fxmacrodata', '', 'EUR1USD', False, '1D', + '2026-01-01', '2026-01-02', 2) + finally: + fxmacrodata.requests.get = original_get + + self.assertTrue(df.empty) if __name__ == '__main__':