differentiable-game.cpp

// differentiable-game.cpp : Defines the entry point for the application.
//

#include "differentiable-game.h"
#include "game.h"
#include "agent.h"
#include "test.h"
#include <iostream>
#include <string_view>
#include <fstream>
#include <filesystem>
#include <spdlog/spdlog.h>
#include <yaml-cpp/yaml.h>
#include <yaml-cpp/node/parse.h>
#include <omp.h>


using namespace std;

void print_range(std::string_view message, auto& range)
{
	cout << message;
	for (const auto& value : range) { cout << value << " "; }
	cout << endl;
}

void simple_func(const Eigen::Vector<float, 5>& test)
{
	std::cout << test << std::endl;
}

namespace YAML
{
	template <>
	struct convert<dg::GameConfig>
	{
		static bool decode(const Node& node, dg::GameConfig& rhs)
		{
			rhs.basic_action_ratio = node["basic_action_ratio"].as<float>();
			rhs.init_goal_range_coeff = node["init_goal_range_coeff"].as<float>();
			rhs.goal_range_multiplier = node["goal_range_multiplier"].as<float>();

			return true;
		}
	};

	template <>
	struct convert<dg::AgentConfig>
	{
		static bool decode(const Node& node, dg::AgentConfig& rhs)
		{
			if (node["device"]) rhs.device = node["device"].as<std::string>();
			if (node["seed"]) rhs.seed = node["seed"].as<long>();
			if (node["n_epochs"]) rhs.n_epochs = node["n_epochs"].as<int>();
			if (node["batch_size"]) rhs.batch_size = node["batch_size"].as<int>();
			if (node["learning_rate"]) rhs.learning_rate = node["learning_rate"].as<float>();
			if (node["epsilon"]) rhs.epsilon = node["epsilon"].as<float>();
			if (node["gradient_steps"]) rhs.gradient_steps = node["gradient_steps"].as<int>();
			if (node["num_episodes_per_train"]) rhs.num_episodes_per_train = node["num_episodes_per_train"].as<int>();
			if (node["num_samples_per_episode"]) rhs.num_samples_per_episode = node["num_samples_per_episode"].as<int>();
			if (node["shared_net_scheme"]) rhs.shared_net_scheme = node["shared_net_scheme"].as<std::vector<int>>();
			if (node["value_net_scheme"]) rhs.value_net_scheme = node["value_net_scheme"].as<std::vector<int>>();
			if (node["policy_net_scheme"]) rhs.policy_net_scheme = node["policy_net_scheme"].as<std::vector<int>>();
			if (node["value_net_scheme"]) rhs.value_net_scheme = node["value_net_scheme"].as<std::vector<int>>();
			if (node["max_episode_length"]) rhs.max_episode_length = node["max_episode_length"].as<int>();
			if (node["epsilon_decay"]) rhs.epsilon_decay = node["epsilon_decay"].as<float>();
			if (node["epsilon_min"]) rhs.epsilon_min = node["epsilon_min"].as<float>();
			if (node["evaluation_period"]) rhs.evaluation_period = node["evaluation_period"].as<int>();
			if (node["num_evaluation_iteration"]) rhs.num_evaluation_iteration = node["num_evaluation_iteration"].as<int>();
			if (node["temperature"]) rhs.temperature = node["temperature"].as<float>();
			if (node["entropy_loss_coefficient"]) rhs.ent_coef = node["entropy_loss_coefficient"].as<float>();
			if (node["value_loss_coefficient"]) rhs.vf_coef = node["value_loss_coefficient"].as<float>();
			if (node["clip_range"]) rhs.clip_range = node["clip_range"].as<float>();

			return true;
		}
	};
}

auto parse_config()
{
	auto config = YAML::LoadFile("config.yaml");
	return std::make_pair(config.as<dg::AgentConfig>(), config["game"].as<dg::GameConfig>());
}

void two_player_value_net_test(const dg::AgentConfig& agent_config, const dg::GameConfig& game_config)
{
	using namespace dg;
	dg::Agent agent(agent_config, game_config);
	agent.learn(1000);
	

	//dg::Agent agent();
}

int main()
{
	// dg::tests::clone_equivalent();

#ifndef NDEBUG
	std::cout << "##### DEBUG BUILD #####" << std::endl;
	// torch::autograd::AnomalyMode::set_enabled(true);
#endif

	// omp_set_num_threads(12);
	// torch::set_num_threads(12);
	torch::init_num_threads();
	std::cout << "torch::hasOpenMP: " << torch::hasOpenMP() << std::endl;
	std::cout << "torch::get_num_threads(): " << torch::get_num_threads() << std::endl;
	std::cout << torch::get_parallel_info() << std::endl;
	
	auto [agent_config, game_config] = parse_config();

	dg::utils::set_random_seed(agent_config.seed);

	two_player_value_net_test(agent_config, game_config);
	
	return 0;
	
	//std::ofstream f("points.csv");
	//
	//dg::Game<3> game(150442840);
	//
	//
	////std::cout << "Init:" << init_state << std::endl;
	//std::cout << "Goal1:" << game.c1().goal() << std::endl;
	//std::cout << "Goal2:" << game.c2().goal() << std::endl;

	//std::default_random_engine rnd{ 1083229 };
	//std::uniform_int_distribution<int> dist(0, decltype(game)::action_space_size - 1);

	//std::vector<std::size_t> timesteps{};
	//float total_reward = 0;

	//for (int i = 0; i < 300; ++i)
	//{
	//	auto init_state = game.reset();
	//	//f << init_state(0) << ", " << init_state(1) << std::endl;
	//	while (!game.done())
	//	{
	//		auto action = dist(rnd);
	//		auto result = game.step(action);
	//		auto& state = std::get<0>(result);
	//		//std::cout << "(" << state(0) << ", " << state(1) << ")" << std::endl;
	//		//f << state(0) << ", " << state(1) << std::endl;
	//		//std::cout << state.norm() << std::endl;
	//		total_reward += std::get<1>(result);
	//	}

	//	std::cout << "timestep: " << game.timestep() << std::endl;
	//	timesteps.push_back(game.timestep());
	//}

	//std::cout << "Total Reward:" << total_reward << std::endl;
	//std::cout << "Average: " << std::accumulate(timesteps.begin(), timesteps.end(), 0.0) / timesteps.size();
	
	return 0;
}