ir_main.c

/*
 * IR - Lightweight JIT Compilation Framework
 * (IR CLI driver)
 * This file is part of the IR Project distributed under the MIT-style LICENSE.
 * Authors: Dmitry Stogov <dmitry@php.net>
 */

#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif

#include "ir.h"
#include "ir_private.h" // TODO: move this together with loader
#include <stdlib.h>
#include <string.h>

#ifdef _WIN32
# include <windows.h>
#elif defined(__linux__) || defined(__sun)
# include <alloca.h>
#endif

#ifndef _WIN32
# include <sys/time.h>
static double ir_time(void)
{
	struct timeval tv;

	gettimeofday(&tv, NULL);
	return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0;
}
#else
static double ir_time(void)
{
	FILETIME filetime;

	GetSystemTimeAsFileTime(&filetime);
	return (double)((((uint64_t)filetime.dwHighDateTime << 32) | (uint64_t)filetime.dwLowDateTime)/10) /
		1000000.0;
}
#endif

static double ir_atexit_start = 0.0;

static void ir_atexit(void)
{
	if (ir_atexit_start) {
		double t = ir_time();
		fprintf(stderr, "\nexecution time = %0.6f\n", t - ir_atexit_start);
		ir_atexit_start = 0.0;
	}
}

static void help(const char *cmd)
{
	printf(
#if HAVE_LLVM
		"Usage: %s [options] [--llvm-bitcode|--llvm-asm] input-file...\n"
#else
		"Usage: %s [options] input-file...\n"
#endif
		"General Options:\n"
		"  --run ...                  - run the main() function of generated code\n"
		"                               (the remaining arguments are passed to main)\n"
		"  --emit-ir [file-name]      - save optimize IR code\n"
		"  -S                         - show generated assembler code\n"
		"  -o <file-name>             - put primary output into the specified file\n"
		"Optimization Options:\n"
		"  -O[012]                    - optimization level (default: -O2)\n"
		"  -f[no-]inline              - enable/disable function inlining (default: enabled at -O1)\n"
		"  -f[no-]tail-calls          - enable/disable tail call optimization (default: enabled at -O1)\n"
		"  -fno-mem2ssa               - disable MEM2SSA pass (default: enabled at -O1)\n"
		"Code Generation Options:\n"
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
		"  -mavx                      - use AVX instruction set\n"
		"  -m[no-]bmi1                - enable/disable BMI1 instruction set\n"
		"  -m[no-]sse3                - enable/disable SSE3 instruction set\n"
		"  -m[no-]ssse3               - enable/disable SSSE3 instruction set\n"
		"  -m[no-]sse4                - enable/disable SSE4 instruction set\n"
		"  -m[no-]sse4.1              - enable/disable SSE4.1 instruction set\n"
		"  -m[no-]sse4.2              - enable/disable SSE4.2 instruction set\n"
		"  -m[no-]avx2                - enable/disable AVX2 instruction set\n"
#endif
		"  -muse-fp                   - use base frame pointer register\n"
#ifndef _WIN32
		"Debugguing Options:\n"
		"  -g                         - produce debugging information (through JITGDB)\n"
		"  -p                         - provide information about JIT-ed code to Linux Perf\n"
		"                               the example usage:\n"
		"                                 $ perf record -k 1 ir -p bench.ir --run\n"
		"                                 $ perf inject -j -i perf.data -o perf.data.jitted\n"
		"                                 $ perf report -i perf.data.jitted\n"
#endif
		"IR Debugging Options:\n"
		"  --save [file-name]         - save IR\n"
		"  --save-cfg                 - save IR with information about CFG\n"
		"  --save-cfg-map             - save IR with information about assigned basic-locks\n"
		"  --save-rules               - save IR with information selectd code-generation \n"
		"  --save-regs                - save IR with information about assigned CPU register\n"
		"  --save-use-lists           - save IR with def->use chains\n"
		"  --save-ir-after-load       - save IR after load and local optimization\n"
		"  --save-ir-after-use-lists  - save IR after USE-LISTS construction\n"
		"  --save-ir-after-mem2ssa    - save IR after MEM2SSA pass\n"
		"  --save-ir-after-sccp       - save IR after SCCP optimization pass\n"
		"  --save-ir-after-cfg        - save IR after CFG construction\n"
		"  --save-ir-after-dom        - save IR after Dominators tree construction\n"
		"  --save-ir-after-loop       - save IR after Loop detection\n"
		"  --save-ir-after-gcm        - save IR after GCM optimization pass\n"
		"  --save-ir-after-scheduling - save IR after SCHEDULE pass\n"
		"  --save-ir-after-matching   - save IR after code selection\n"
		"  --save-ir-after-live-ranges - save IR after live ranges identification\n"
		"  --save-ir-after-coalescing - save IR after live ranges coalescing\n"
		"  --save-ir-after-regalloc   - save IR after register allocation\n"
		"  --save-ir-codegen          - save scheduled IR with selected rules and assigned CPU registers\n"
		"  --save-ir-final            - save IR after all passes\n"
		"  --save-ir-after-each-pass  - save IR after each pass\n"
		"  --save-live-ranges         - save info about live ranges (use with --save-ir-after-live-ranges)\n"
		"  --save-dot [file-name]     - save IR in .DOT format (affects all --save-ir-...)\n"
		"                               the output may be converted into multi-page PDF using pipe: \n"
		"                                 $ ir ... 2>&1 | dot -Tps:cairo:cairo | ps2pdf - > out.pdf\n"
#ifdef IR_DEBUG
		"  --debug-sccp               - debug SCCP optimization pass\n"
		"  --debug-gcm                - debug GCM optimization pass\n"
		"  --debug-gcm-split          - debug floating node splitting\n"
		"  --debug-scheduling         - debug SCHEDULE optimization pass\n"
		"  --debug-regalloc           - debug register allocator\n"
		"  --debug-regset <bit-mask>  - restrict available register set\n"
		"  --debug-bb-scheduling      - debug BB PLCEMENT optimization pass\n"
#endif
		"Utility Options\n"
		"  -fsyntax-only              - check the input files for syntax errors, but don't do anything beyond that\n"
		"  --emit-c                   - convert final IR to C source (implementation is incomplete)\n"
		"  --emit-llvm                - convert final IR to LLVM code (implementation is incomplete)\n"
		"  --dump-size                - dump generated code size\n"
		"  --dump-time                - dump compilation and execution time\n"
		"  --target                   - print JIT target\n"
		"  --version\n"
		"  --help\n",
		cmd);
}

#define IR_DUMP_IR                  (1<<0)
#define IR_DUMP_LIVE_RANGES         (1<<1)
#define IR_DUMP_DOT                 (1<<2)

#define IR_DUMP_C                   (1<<3)
#define IR_DUMP_LLVM                (1<<4)
#define IR_DUMP_ASM                 (1<<5)
#define IR_DUMP_SIZE                (1<<6)

#define IR_RUN                      (1<<7)
#define IR_PERF                     (1<<8)
#define IR_GDB                      (1<<9)

#define IR_LOAD_ONLY                (1<<10)

#define IR_GEN_NATIVE               (IR_RUN|IR_DUMP_ASM|IR_DUMP_SIZE)
#define IR_GEN_CODE                 (IR_DUMP_LLVM|IR_DUMP_C)

#define IR_DUMP_AFTER_LOAD          (1<<16)
#define IR_DUMP_AFTER_USE_LISTS     (1<<17)
#define IR_DUMP_AFTER_MEM2SSA       (1<<18)
#define IR_DUMP_AFTER_SCCP          (1<<19)
#define IR_DUMP_AFTER_CFG           (1<<20)
#define IR_DUMP_AFTER_DOM           (1<<21)
#define IR_DUMP_AFTER_LOOP          (1<<22)
#define IR_DUMP_AFTER_GCM           (1<<23)
#define IR_DUMP_AFTER_SCHEDULING    (1<<24)
#define IR_DUMP_AFTER_CODE_MATCHING (1<<25)
#define IR_DUMP_AFTER_LIVE_RANGES   (1<<26)
#define IR_DUMP_AFTER_COALESCING    (1<<27)
#define IR_DUMP_AFTER_REGALLOC      (1<<28)
#define IR_DUMP_FINAL               (1<<29)

#define IR_DUMP_CODEGEN             (1<<30)

#define IR_DUMP_AFTER_ALL           0x3fff0000

#define IR_UNKNOWN_SIZE             1

static int _save(ir_ctx *ctx, uint32_t save_flags, uint32_t dump, uint32_t pass, FILE *f, const char *func_name)
{
	const char *comments = NULL;

#if 0
	bool close = 0;

	if (!f) {
		char fn[4096];

		if (dump & IR_DUMP_AFTER_ALL) {
			if (pass == IR_DUMP_AFTER_LOAD) {
				snprintf(fn, sizeof(fn)-1, "01-load-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_USE_LISTS) {
				snprintf(fn, sizeof(fn)-1, "02-use-lists-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_MEM2SSA) {
				snprintf(fn, sizeof(fn)-1, "03-mem2ssa-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_SCCP) {
				snprintf(fn, sizeof(fn)-1, "04-sccp-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_CFG) {
				snprintf(fn, sizeof(fn)-1, "05-cfg-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_DOM) {
				snprintf(fn, sizeof(fn)-1, "06-dom-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_LOOP) {
				snprintf(fn, sizeof(fn)-1, "07-loop-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_GCM) {
				snprintf(fn, sizeof(fn)-1, "08-gcm-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_SCHEDULING) {
				snprintf(fn, sizeof(fn)-1, "09-scheduling-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_LIVE_RANGES) {
				snprintf(fn, sizeof(fn)-1, "10-live-ranges-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_COALESCING) {
				snprintf(fn, sizeof(fn)-1, "11-coalescing-%s.ir", func_name);
			} else if (pass == IR_DUMP_AFTER_REGALLOC) {
				snprintf(fn, sizeof(fn)-1, "12-regalloc-%s.ir", func_name);
			} else if (pass == IR_DUMP_FINAL) {
				if (dump & IR_DUMP_CODEGEN) {
					snprintf(fn, sizeof(fn)-1, "13-codegen-%s.ir", func_name);
				} else {
					snprintf(fn, sizeof(fn)-1, "13-final-%s.ir", func_name);
				}
			} else {
				f = stderr; // TODO:
			}
		} else {
			snprintf(fn, sizeof(fn)-1, "%s.ir", func_name);
		}
		f = fopen(fn, "w+");
		if (!f) {
			fprintf(stderr, "ERROR: Cannot create file '%s'\n", fn);
			return 0;
		}
		close = 1;
	}
#endif

	if ((dump & IR_DUMP_DOT)
	 || ((dump & (IR_DUMP_IR|IR_DUMP_CODEGEN|IR_DUMP_LIVE_RANGES))
	  && !IR_IS_POWER_OF_TWO(dump & (IR_DUMP_AFTER_ALL|IR_DUMP_CODEGEN)))) {
		if (pass == IR_DUMP_AFTER_LOAD) {
			comments = "(after load)";
		} else if (pass == IR_DUMP_AFTER_USE_LISTS) {
			comments = "(after use lists)";
		} else if (pass == IR_DUMP_AFTER_MEM2SSA) {
			comments = "(after mem2ssa)";
		} else if (pass == IR_DUMP_AFTER_SCCP) {
			comments = "(after sccp)";
		} else if (pass == IR_DUMP_AFTER_CFG) {
			comments = "(after cfg)";
		} else if (pass == IR_DUMP_AFTER_DOM) {
			comments = "(after dom)";
		} else if (pass == IR_DUMP_AFTER_LOOP) {
			comments = "(after loop)";
		} else if (pass == IR_DUMP_AFTER_GCM) {
			comments = "(after gcm)";
		} else if (pass == IR_DUMP_AFTER_SCHEDULING) {
			comments = "(after scheduling)";
		} else if (pass == IR_DUMP_AFTER_CODE_MATCHING) {
			comments = "(after code mathing)";
		} else if (pass == IR_DUMP_AFTER_LIVE_RANGES) {
			comments = "(after live ranges)";
		} else if (pass == IR_DUMP_AFTER_COALESCING) {
			comments = "(after coalesing)";
		} else if (pass == IR_DUMP_AFTER_REGALLOC) {
			comments = "(after regalloc)";
		} else if (pass == IR_DUMP_FINAL) {
			if (dump & IR_DUMP_CODEGEN) {
				comments = "(codegen)";
			} else if (!(dump & IR_DUMP_DOT)) {
				comments = "(final)";
			}
		}
	}
	if (dump & IR_DUMP_DOT) {
		ir_dump_dot(ctx, func_name, comments, f);
	} else {
		if (comments) {
			fprintf(f, "# %s\n", comments);
		}
		if (pass == IR_DUMP_FINAL && (dump & IR_DUMP_CODEGEN)) {
			ir_dump_codegen(ctx, f);
		} else if (dump & (IR_DUMP_IR|IR_DUMP_AFTER_ALL)) {
			ir_save(ctx, save_flags, f);
		}
		if (dump & IR_DUMP_LIVE_RANGES) {
			ir_dump_live_ranges(ctx, f);
		}
	}

#if 0
	if (close) {
		fclose(f);
	}
#endif

	return 1;
}

int ir_compile_func(ir_ctx *ctx, int opt_level, uint32_t save_flags, uint32_t dump, FILE *dump_file, const char *func_name)
{
	if ((dump & IR_DUMP_AFTER_LOAD)
	 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_LOAD, dump_file, func_name)) {
		return 0;
	}

	if (opt_level > 0 || (dump & (IR_GEN_NATIVE|IR_GEN_CODE))) {
		ir_build_def_use_lists(ctx);
		if ((dump & IR_DUMP_AFTER_USE_LISTS)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_USE_LISTS, dump_file, func_name)) {
			return 0;
		}
	}

#ifdef IR_DEBUG
	if (!ir_check(ctx)) return 0;
#endif

	if (dump & IR_LOAD_ONLY) {
		return 1;
	}

	if (opt_level > 0 && (ctx->flags & IR_OPT_MEM2SSA)) {
		ir_build_cfg(ctx);
		if ((dump & IR_DUMP_AFTER_CFG)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_CFG, dump_file, func_name)) {
			return 0;
		}
#ifdef IR_DEBUG
		if (!ir_check(ctx)) return 0;
#endif

		ir_build_dominators_tree(ctx);
		if ((dump & IR_DUMP_AFTER_DOM)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_DOM, dump_file, func_name)) {
			return 0;
		}

		ir_mem2ssa(ctx);
		ir_reset_cfg(ctx);

		if ((dump & IR_DUMP_AFTER_MEM2SSA)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_MEM2SSA, dump_file, func_name)) {
			return 0;
		}
#ifdef IR_DEBUG
		if (!ir_check(ctx)) return 0;
#endif
	}

	/* Global Optimization */
	if (opt_level > 1) {
		ir_sccp(ctx);
		if ((dump & IR_DUMP_AFTER_SCCP)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_SCCP, dump_file, func_name)) {
			return 0;
		}
#ifdef IR_DEBUG
		if (!ir_check(ctx)) return 0;
#endif
	}

	if ((opt_level > 0 || (dump & (IR_GEN_NATIVE|IR_GEN_CODE))) && !ctx->cfg_blocks) {
		ir_build_cfg(ctx);
		if ((dump & IR_DUMP_AFTER_CFG)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_CFG, dump_file, func_name)) {
			return 0;
		}
#ifdef IR_DEBUG
		if (!ir_check(ctx)) return 0;
#endif

		if (opt_level > 0) {
			ir_build_dominators_tree(ctx);
			if ((dump & IR_DUMP_AFTER_DOM)
			 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_DOM, dump_file, func_name)) {
				return 0;
			}
		}
	}

	/* Schedule */
	if (opt_level > 0) {
		ir_find_loops(ctx);
		if ((dump & IR_DUMP_AFTER_LOOP)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_LOOP, dump_file, func_name)) {
			return 0;
		}

		ir_gcm(ctx);
		if ((dump & IR_DUMP_AFTER_GCM)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_GCM, dump_file, func_name)) {
			return 0;
		}
#ifdef IR_DEBUG
		if (!ir_check(ctx)) return 0;
#endif

		ir_schedule(ctx);
		if ((dump & IR_DUMP_AFTER_SCHEDULING)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_SCHEDULING, dump_file, func_name)) {
			return 0;
		}
#ifdef IR_DEBUG
		if (!ir_check(ctx)) return 0;
#endif
	}

	if (dump & IR_GEN_NATIVE) {
		ctx->func_name = ir_string(ctx, func_name);

		ir_match(ctx);

		if ((dump & IR_DUMP_AFTER_CODE_MATCHING)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_CODE_MATCHING, dump_file, func_name)) {
			return 0;
		}
	}

	if (opt_level > 0) {
		ir_assign_virtual_registers(ctx);
		ir_compute_live_ranges(ctx);

		if ((dump & IR_DUMP_AFTER_LIVE_RANGES)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_LIVE_RANGES, dump_file, func_name)) {
			return 0;
		}

		ir_coalesce(ctx);

		if ((dump & IR_DUMP_AFTER_COALESCING)
		 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_COALESCING, dump_file, func_name)) {
			return 0;
		}

		if (dump & IR_GEN_NATIVE) {
			ir_reg_alloc(ctx);
			if ((dump & IR_DUMP_AFTER_REGALLOC)
			 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_REGALLOC, dump_file, func_name)) {
				return 0;
			}
		}

		ir_schedule_blocks(ctx);
	} else if (dump & (IR_GEN_NATIVE|IR_DUMP_C)) {
		ir_assign_virtual_registers(ctx);
		if (dump & IR_GEN_NATIVE) {
			ir_reg_alloc_O0(ctx);
			if ((dump & IR_DUMP_AFTER_REGALLOC)
			 && !_save(ctx, save_flags, dump, IR_DUMP_AFTER_REGALLOC, dump_file, func_name)) {
				return 0;
			}
		} else {
			ir_compute_dessa_moves(ctx);
		}
	}

	if ((dump & (IR_DUMP_FINAL|IR_DUMP_CODEGEN))
	 && !_save(ctx, save_flags, dump, IR_DUMP_FINAL, dump_file, func_name)) {
		return 0;
	}

#ifdef IR_DEBUG
	if (!ir_check(ctx)) return 0;
#endif

	return 1;
}

typedef struct _ir_sym {
	void *addr;
	void *thunk_addr;
} ir_sym;

typedef struct _ir_reloc {
	void   *addr;
	ir_str  sym;
} ir_reloc;

typedef struct _ir_main_loader {
	ir_loader  loader;
	int        opt_level;
	uint32_t   mflags;
	uint64_t   debug_regset;
	uint32_t   save_flags;
	uint32_t   dump;
	size_t     size;
	void      *main;
	FILE      *dump_file;
	FILE      *out_file;
	ir_strtab  symtab;
	ir_sym    *sym;
	ir_reloc  *reloc;
	ir_str     sym_count;
	uint32_t   reloc_count;
	void      *data_start;
	size_t     data_pos;
	ir_code_buffer code_buffer;
} ir_main_loader;

static void ir_loader_free_symbols(ir_main_loader *l)
{
	ir_strtab_free(&l->symtab);
	if (l->sym) {
		ir_mem_free(l->sym);
	}
	if (l->reloc) {
		ir_mem_free(l->reloc);
	}
}

static void ir_loader_add_reloc(ir_main_loader *l, const char *name, void *addr)
{
	ir_reloc *r;
	ir_str val = ir_strtab_count(&l->symtab) + 1;
	ir_str sym = ir_strtab_lookup(&l->symtab, name, strlen(name), val);

	if (sym == val) {
		if (val >= l->sym_count) {
			l->sym_count += 16;
			l->sym = ir_mem_realloc(l->sym, sizeof(ir_sym) * l->sym_count);
		}
		l->sym[val].addr = NULL;
		l->sym[val].thunk_addr = NULL;
	}

	l->reloc = ir_mem_realloc(l->reloc, sizeof(ir_reloc) * (l->reloc_count + 1));
	r = &l->reloc[l->reloc_count];
	r->addr = addr;
	r->sym = sym;
	l->reloc_count++;
}

static bool ir_loader_fix_relocs(ir_main_loader *l)
{
	bool ret = 1;
	uint32_t n = l->reloc_count;

	if (n > 0) {
		ir_reloc *r = l->reloc;
		ir_sym *s;

		ir_mem_unprotect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
		for (; n > 0; r++, n--) {
			IR_ASSERT(r->sym > 0 && r->sym < l->sym_count);
			s = &l->sym[r->sym];
			if (s->addr) {
				*(void**)r->addr = (void*)((uintptr_t)s->addr + *(uintptr_t*)r->addr);
#if 0
				uintptr_t addr;

				memcpy(&addr, r->addr, sizeof(void*));
				addr += (uintptr_t)s->addr;
				memcpy(r->addr, &addr, sizeof(void*));
#endif
			} else {
				fprintf(stderr, "Undefined symbol: %s\n", ir_strtab_str(&l->symtab, r->sym - 1));
				ret = 0;
				break;
			}
		}
		ir_mem_protect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
	}

	if (ret) {
		/* Check for unresolved external symbols */
		n = ir_strtab_count(&l->symtab);
		if (n > 0) {
			ir_sym *s = l->sym + 1;
			uint32_t j;

			for (j = 1; j < n; s++, j++) {
				if (!s->addr && s->thunk_addr) {
					fprintf(stderr, "Undefined symbol: %s\n", ir_strtab_str(&l->symtab, j - 1));
					ret = 0;
				}
			}
		}
	}

	return ret;
}

static bool ir_loader_add_sym(ir_loader *loader, const char *name, void *addr)
{
	ir_main_loader *l = (ir_main_loader*)loader;
	uint32_t len = (uint32_t)strlen(name);
	ir_str val = ir_strtab_count(&l->symtab) + 1;
	ir_str old_val = ir_strtab_lookup(&l->symtab, name, len, val);

	if (old_val != val) {
		if (addr && !l->sym[old_val].addr) {
			/* Update forward declaration */
			l->sym[old_val].addr = addr;
			if (l->sym[old_val].thunk_addr) {
				// TODO: Fix thunk or relocation ???
				if (l->code_buffer.start) {
					ir_mem_unprotect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
				}
				ir_fix_thunk(l->sym[old_val].thunk_addr, addr);
				if (l->code_buffer.start) {
					ir_mem_protect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
				}
			}
			return 1;
		}
		return 0;
	}
	if (val >= l->sym_count) {
		l->sym_count += 16;
		l->sym = ir_mem_realloc(l->sym, sizeof(ir_sym) * l->sym_count);
	}
	l->sym[val].addr = addr;
	l->sym[val].thunk_addr = NULL;
	return 1;
}

static bool ir_loader_has_sym(ir_loader *loader, const char *name)
{
	ir_main_loader *l = (ir_main_loader*)loader;
	uint32_t len = (uint32_t)strlen(name);
	ir_ref val = ir_strtab_find(&l->symtab, name, len);
	return val != 0;
}

static void* ir_loader_resolve_sym_name(ir_loader *loader, const char *name, uint32_t flags)
{
	ir_main_loader *l = (ir_main_loader*)loader;
	uint32_t len = (uint32_t)strlen(name);
	ir_ref val = ir_strtab_find(&l->symtab, name, len);
	void *addr;

	if (val) {
		if (l->sym[val].addr) {
			return l->sym[val].addr;
		}
		if (!l->sym[val].thunk_addr && (flags & IR_RESOLVE_SYM_ADD_THUNK)) {
			/* Undefined declaration */
			// TODO: Add thunk or relocation ???
			size_t size;

			l->sym[val].thunk_addr = ir_emit_thunk(&l->code_buffer, NULL, &size);
			ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)l->sym[val].thunk_addr, size);
		}
		return l->sym[val].thunk_addr;
	}
	addr = ir_resolve_sym_name(name);
	if (addr) {
		ir_loader_add_sym(loader, name, addr); /* cache */
	}
	if (!addr && !(flags & IR_RESOLVE_SYM_SILENT)) {
		fprintf(stderr, "Undefined symbol: %s\n", name);
	}
	return addr;
}

static void* _ir_loader_resolve_sym_name(ir_loader *loader, ir_ctx *ctx, ir_str name, uint32_t flags)
{
	return ir_loader_resolve_sym_name(loader, ir_get_str(ctx, name), flags);
}

static bool ir_loader_external_sym_dcl(ir_loader *loader, const char *name, uint32_t flags)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	if (ir_loader_has_sym(loader, name)) {
		return 1;
	}

	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		fprintf(l->dump_file, "extern %s %s%s;\n",
			(flags & IR_CONST) ? "const" : "var",
			(l->save_flags & IR_SAVE_SAFE_NAMES) ? "@" : "",
			name);
	}
	if (l->dump & IR_DUMP_C) {
		ir_emit_c_sym_decl(name, flags | IR_EXTERN, l->out_file);
	}
	if (l->dump & IR_DUMP_LLVM) {
		ir_emit_llvm_sym_decl(name, flags | IR_EXTERN, l->out_file);
	}
	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		void *addr = ir_loader_resolve_sym_name(loader, name, IR_RESOLVE_SYM_SILENT);

		if (!addr) {
			return 0;
		}
		if (l->dump & IR_DUMP_ASM) {
			ir_disasm_add_symbol(name, (uintptr_t)addr, IR_UNKNOWN_SIZE);
		}
	} else {
		ir_loader_add_sym(loader, name, NULL);
	}
	return 1;
}

static void ir_dump_func_dcl(const char *name, uint32_t flags, ir_type ret_type, uint32_t params_count, const uint8_t *param_types, FILE *f, uint32_t save_flags)
{
	if (flags & IR_EXTERN) {
		fprintf(f, "extern ");
	} else if (flags & IR_STATIC) {
		fprintf(f, "static ");
	}
	fprintf(f, "func %s%s",
		(save_flags & IR_SAVE_SAFE_NAMES) ? "@" : "",
		name);
	ir_print_proto_ex(flags, ret_type, params_count, param_types, f);
	fprintf(f, ";\n");
}

static bool ir_loader_external_func_dcl(ir_loader *loader, const char *name, uint32_t flags,
                                        ir_type ret_type, uint32_t params_count, const uint8_t *param_types)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	if (ir_loader_has_sym(loader, name)) {
		return 1;
	}

	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		ir_dump_func_dcl(name, flags | IR_EXTERN, ret_type, params_count, param_types, l->dump_file, l->save_flags);
	}
	if (l->dump & IR_DUMP_C) {
		ir_emit_c_func_decl(name, flags | IR_EXTERN, ret_type,  params_count, param_types, l->out_file);
	}
	if (l->dump & IR_DUMP_LLVM) {
		ir_emit_llvm_func_decl(name, flags | IR_EXTERN, ret_type,  params_count, param_types, l->out_file);
	}
	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		void *addr = ir_loader_resolve_sym_name(loader, name, IR_RESOLVE_SYM_SILENT);

		if (!addr) {
			/* Unresolved external function */
			ir_loader_add_sym(loader, name, NULL);
			return 1;
		}
		if (l->dump & IR_DUMP_ASM) {
			ir_disasm_add_symbol(name, (uintptr_t)addr, IR_UNKNOWN_SIZE);
		}
	} else {
		ir_loader_add_sym(loader, name, NULL);
	}
	return 1;
}

static bool ir_loader_forward_func_dcl(ir_loader *loader, const char *name, uint32_t flags,
                                       ir_type ret_type, uint32_t params_count, const uint8_t *param_types)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	if (ir_loader_has_sym(loader, name)) {
		return 1;
	}

	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		ir_dump_func_dcl(name, flags, ret_type, params_count, param_types, l->dump_file, l->save_flags);
	}
	if (l->dump & IR_DUMP_C) {
		ir_emit_c_func_decl(name, flags, ret_type,  params_count, param_types, l->out_file);
	}

	ir_loader_add_sym(loader, name, NULL);
	return 1;
}

static bool ir_loader_sym_dcl(ir_loader *loader, const char *name, uint32_t flags, size_t size)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	l->data_start = NULL;
	l->data_pos = 0;

	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		if (flags & IR_STATIC) {
			fprintf(l->dump_file, "static ");
		}
		fprintf(l->dump_file, "%s %s%s[%" PRIuPTR "]%s",
			(flags & IR_CONST) ? "const" : "var",
			(l->save_flags & IR_SAVE_SAFE_NAMES) ? "@" : "",
			name,
			size,
			(flags & IR_INITIALIZED) ? ((flags & IR_CONST_STRING) ? " = " : " = {\n") : ";\n");
	}
	if (l->dump & IR_DUMP_C) {
		ir_emit_c_sym_decl(name, flags, l->out_file);
	}
	if (l->dump & IR_DUMP_LLVM) {
		ir_emit_llvm_sym_decl(name, flags, l->out_file);
	}
	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		void *data;

		if (flags & IR_CONST) {
			data = l->code_buffer.pos;
			/* Data Alignment */
			if (size > 8) {
				data = (void*)IR_ALIGNED_SIZE(((size_t)(data)), 16);
			} else if (size == 8) {
				data = (void*)IR_ALIGNED_SIZE(((size_t)(data)), 8);
			} else if (size >= 4) {
				data = (void*)IR_ALIGNED_SIZE(((size_t)(data)), 4);
			} else if (size >= 2) {
				data = (void*)IR_ALIGNED_SIZE(((size_t)(data)), 2);
			}
			if (size > (size_t)((char*)l->code_buffer.end - (char*)data)) {
				return 0;
			}
			l->code_buffer.pos = (char*)data + size;
			ir_mem_unprotect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
		} else {
			data = ir_mem_malloc(size);
		}

		if (!ir_loader_add_sym(loader, name, data)) {
			if (flags & IR_CONST) {
				l->code_buffer.pos = data;
			} else {
				ir_mem_free(data);
			}
			return 0;
		}
		memset(data, 0, size);
		if (flags & IR_INITIALIZED) {
			l->data_start = data;
		}
		if (l->dump & IR_DUMP_ASM) {
			ir_disasm_add_symbol(name, (uintptr_t)data, size);
		}
#ifndef _WIN32
		if (l->dump & IR_GDB) {
			ir_gdb_register(name, data, size, 0, 0);
		}
#endif
	} else {
		ir_loader_add_sym(loader, name, NULL);
	}
	return 1;
}

static bool ir_loader_sym_data(ir_loader *loader, ir_type type, uint32_t count, const void *data)
{
	ir_main_loader *l = (ir_main_loader*) loader;
	size_t size;

	IR_ASSERT(IR_IS_TYPE_SCALAR(type));
	size = ir_type_size[type];
	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		const void *p = data;
		uint32_t i;

		switch (size) {
			case 1:
				for (i = 0; i < count; i++) {
					if (!*(uint8_t*)p) {
						fprintf(l->dump_file, "\t%s 0,\n", ir_type_cname[type]);
					} else {
						fprintf(l->dump_file, "\t%s 0x%02x,\n", ir_type_cname[type], (uint32_t)*(uint8_t*)p);
					}
				}
				break;
			case 2:
				for (i = 0; i < count; i++) {
					if (!*(uint16_t*)p) {
						fprintf(l->dump_file, "\t%s 0,\n", ir_type_cname[type]);
					} else {
						fprintf(l->dump_file, "\t%s 0x%04x,\n", ir_type_cname[type], (uint32_t)*(uint16_t*)p);
					}
				}
				break;
			case 4:
				for (i = 0; i < count; i++) {
					if (!*(uint32_t*)p) {
						fprintf(l->dump_file, "\t%s 0,\n", ir_type_cname[type]);
					} else {
						fprintf(l->dump_file, "\t%s 0x%08x,\n", ir_type_cname[type], *(uint32_t*)p);
					}
				}
				break;
			case 8:
				for (i = 0; i < count; i++) {
					if (!*(uint64_t*)p) {
						fprintf(l->dump_file, "\t%s 0,\n", ir_type_cname[type]);
					} else {
						fprintf(l->dump_file, "\t%s 0x%016" PRIx64 ",\n", ir_type_cname[type], *(uint64_t*)p);
					}
				}
				break;
		}
	}
	if (l->dump & IR_DUMP_C) {
		// TODO:
	}
	if (l->dump & IR_DUMP_LLVM) {
		// TODO:
	}
	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		IR_ASSERT(l->data_start);
		if (count == 1) {
			memcpy((char*)l->data_start + l->data_pos, data, size);
		} else {
			size_t pos = 0;
			uint32_t i;

			IR_ASSERT(count > 1);
			for (i = 0; i < count; i++) {
				memcpy((char*)l->data_start + l->data_pos + pos, data, size);
				pos += size;
			}
		}
	}
	l->data_pos += size * count;
	return 1;
}

static bool ir_loader_sym_data_str(ir_loader *loader, const char *str, size_t len)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		fprintf(l->dump_file, "\"");
		ir_print_escaped_str(str, len, l->dump_file);
		fprintf(l->dump_file, "\"");
	}
	if (l->dump & IR_DUMP_C) {
		// TODO:
	}
	if (l->dump & IR_DUMP_LLVM) {
		// TODO:
	}
	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		IR_ASSERT(l->data_start);
		memcpy((char*)l->data_start + l->data_pos, str, len);
	}
	l->data_pos += len;
	return 1;
}

static bool ir_loader_sym_data_pad(ir_loader *loader, size_t offset)
{
	ir_main_loader *l = (ir_main_loader*) loader;
	size_t i;

	IR_ASSERT(offset >= l->data_pos);
	offset -= l->data_pos;
	if (offset) {
		if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
			for (i = 0; i < offset; i++) {
				fprintf(l->dump_file, "\tuint8_t 0x00,\n");
			}
		}
		if (l->dump & IR_DUMP_C) {
			// TODO:
		}
		if (l->dump & IR_DUMP_LLVM) {
			// TODO:
		}
		if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
			IR_ASSERT(l->data_start);
			memset((char*)l->data_start + l->data_pos, 0, offset);
		}
		l->data_pos += offset;
	}
	return 1;
}

static bool ir_loader_sym_data_ref(ir_loader *loader, ir_op op, const char *ref, uintptr_t offset)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	IR_ASSERT(op == IR_FUNC || op == IR_SYM);
	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		if (!offset) {
			fprintf(l->dump_file, "\t%s %s(%s%s),\n",
				ir_type_cname[IR_ADDR],
				op == IR_FUNC ? "func" : "sym",
				(l->save_flags & IR_SAVE_SAFE_NAMES) ? "@" : "",
				ref);
		} else {
			fprintf(l->dump_file, "\t%s %s(%s%s)+0x%" PRIxPTR ",\n",
				ir_type_cname[IR_ADDR],
				op == IR_FUNC ? "func" : "sym",
				(l->save_flags & IR_SAVE_SAFE_NAMES) ? "@" : "",
				ref,
				offset);
		}
	}
	if (l->dump & IR_DUMP_C) {
		// TODO:
	}
	if (l->dump & IR_DUMP_LLVM) {
		// TODO:
	}
	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		void *data = (char*)l->data_start + l->data_pos;
		void *addr = ir_loader_resolve_sym_name(loader, ref, IR_RESOLVE_SYM_SILENT);

		if (!addr) {
			ir_loader_add_reloc(l, ref, data);
		}
		IR_ASSERT(l->data_start);
		addr = (void*)((uintptr_t)(addr) + offset);
		memcpy(data, &addr, sizeof(void*));
	}
	l->data_pos += sizeof(void*);
	return 1;
}

static bool ir_loader_sym_data_end(ir_loader *loader, uint32_t flags)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	if ((l->dump & IR_DUMP_IR) && (l->dump_file)) {
		if (flags & IR_CONST_STRING) {
			fprintf(l->dump_file, ";\n");
		} else {
			fprintf(l->dump_file, "};\n");
		}
	}
	if (l->dump & IR_DUMP_C) {
		// TODO:
	}
	if (l->dump & IR_DUMP_LLVM) {
		// TODO:
	}
	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		if ((char*)l->data_start >= (char*)l->code_buffer.start
		 && (char*)l->data_start < (char*)l->code_buffer.end) {
			ir_mem_protect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
		}
		// TODO:
	}
	return 1;
}

static bool ir_loader_func_init(ir_loader *loader, ir_ctx *ctx, const char *name)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	ir_init(ctx, loader->default_func_flags, 256, 1024);
	ctx->mflags = l->mflags;
	ctx->fixed_regset = ~l->debug_regset;
	ctx->loader = loader;
	return 1;
}

static bool ir_loader_func_process(ir_loader *loader, ir_ctx *ctx, const char *name)
{
	ir_main_loader *l = (ir_main_loader*) loader;

	// TODO: remove this
	if (ctx->ret_type == (ir_type)-1 && ctx->insns_count > 1 && ctx->ir_base[1].op == IR_START) {
		ir_ref ref = ctx->ir_base[1].op1;
		while (ref) {
			ir_insn *insn = &ctx->ir_base[ref];
			if (insn->op == IR_RETURN) {
				ctx->ret_type = insn->op2 ? ctx->ir_base[insn->op2].type : IR_VOID;
				break;
			} else if (insn->op == IR_UNREACHABLE && ctx->ir_base[insn->op1].op == IR_TAILCALL) {
				ctx->ret_type = ctx->ir_base[insn->op1].type;
				break;
			}
			ref = ctx->ir_base[ref].op3;
		}
	}

	if (name == NULL) {
		name = (l->dump & IR_RUN) ? "main" : "test";
	} else if ((l->dump & (IR_DUMP_IR|IR_DUMP_CODEGEN|IR_DUMP_LIVE_RANGES)) && l->dump_file) {
		ir_print_func_proto(ctx, name, (l->save_flags & IR_SAVE_SAFE_NAMES) != 0, l->dump_file);
		fprintf(l->dump_file, "\n");
	}

	if (!ir_compile_func(ctx, l->opt_level, l->save_flags, l->dump, l->dump_file, name)) {
		return 0;
	}

	if (l->dump & IR_DUMP_C) {
		if (!ir_emit_c(ctx, name, l->out_file)) {
			fprintf(stderr, "\nERROR: %d\n", ctx->status);
			return 0;
		}
	}

	if (l->dump & IR_DUMP_LLVM) {
		if (!ir_emit_llvm(ctx, name, l->out_file)) {
			fprintf(stderr, "\nERROR: %d\n", ctx->status);
			return 0;
		}
	}

	if (l->dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		size_t size;
		void *entry;

		if (l->code_buffer.start) {
			ctx->code_buffer = &l->code_buffer;
			ir_mem_unprotect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
		}
		entry = ir_emit_code(ctx, &size);
#ifndef _WIN32
		if (l->dump & IR_GDB) {
			if (!l->code_buffer.start) {
				ir_mem_unprotect(entry, size);
			}
			ir_gdb_register(name, entry, size, sizeof(void*), 0);
			if (!l->code_buffer.start) {
				ir_mem_protect(entry, size);
			}
		}
#endif
		if (l->code_buffer.start) {
			ir_mem_protect(l->code_buffer.start, (char*)l->code_buffer.end - (char*)l->code_buffer.start);
		}
		if (entry) {
			if (!l->code_buffer.start) {
				l->size += size;
				l->size = IR_ALIGNED_SIZE(l->size, 16);
			}
			if (!ir_loader_add_sym(loader, name, entry)) {
				fprintf(stderr, "\nERROR: Symbol redefinition: %s\n", name);
				return 0;
			}
			if (l->dump & IR_DUMP_ASM) {
				ir_ref i;
				ir_insn *insn;

				ir_disasm_add_symbol(name, (uintptr_t)entry, size);

				for (i = IR_UNUSED + 1, insn = ctx->ir_base - i; i < ctx->consts_count; i++, insn--) {
					if (insn->op == IR_FUNC) {
						const char *name = ir_get_str(ctx, insn->val.name);
						void *addr = ir_loader_resolve_sym_name(loader, name, 0);

						IR_ASSERT(addr);
						ir_disasm_add_symbol(name, (uintptr_t)addr, IR_UNKNOWN_SIZE);
//TODO:					} else if (insn->op == IR_SYM) {
					}
				}

				ir_disasm(name, entry, size, 0, ctx, l->out_file);
			}
			if (l->dump & IR_RUN) {
#ifndef _WIN32
				if (l->dump & IR_PERF) {
					ir_perf_map_register(name, entry, size);
					ir_perf_jitdump_register(name, entry, size);
				}
#endif
				if (strcmp(name, "main") == 0) {
					l->main = entry;
				}
			}
		} else {
			fprintf(stderr, "\nERROR: %d\n", ctx->status);
			return 0;
		}
	} else {
		ir_loader_add_sym(loader, name, NULL);
	}
	return 1;
}

#ifdef FUZZ_LIBFUZZER
int _fuzz_main(int argc, char **argv)
#else
int main(int argc, char **argv)
#endif
{
	int i, run_args = 0;
	char *input = NULL;
	char *dump_file = NULL, *out_file = 0;
	FILE *f;
	bool dump_time = 0;
	bool disable_inline = 0;
	bool force_inline = 0;
	bool disable_tail_calls = 0;
	bool force_tail_calls = 0;
	bool disable_mem2ssa = 0;
	uint32_t save_flags = 0;
	uint32_t dump = 0;
	int opt_level = 2;
	uint32_t flags = 0;
	uint32_t mflags = 0;
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
	uint32_t mflags_disabled = 0;
#endif
	uint64_t debug_regset = 0xffffffffffffffff;
#ifdef _WIN32
	bool abort_fault = 1;
#endif
#if HAVE_LLVM
	bool load_llvm_bitcode = 0;
	bool load_llvm_asm = 0;
#endif
	ir_main_loader loader;
	int ret = 0;
	double start;

	ir_consistency_check();

	for (i = 1; i < argc; i++) {
		if (strcmp(argv[i], "-h") == 0
		 || strcmp(argv[i], "--help") == 0) {
			help(argv[0]);
			return 0;
		} else if (strcmp(argv[i], "--version") == 0) {
			printf("IR %s\n", IR_VERSION);
			return 0;
		} else if (strcmp(argv[i], "--target") == 0) {
			printf("%s\n", IR_TARGET_TRIPLET);
			return 0;
		} else if (argv[i][0] == '-' && argv[i][1] == 'O' && strlen(argv[i]) == 3) {
			if (argv[i][2] == '0') {
				opt_level = 0;
			} else if (argv[i][2] == '1') {
				opt_level = 1;
			} else if (argv[i][2] == '2') {
				opt_level = 2;
			} else {
				fprintf(stderr, "ERROR: Invalid usage' (use --help)\n");
				return 1;
			}
		} else if (argv[i][0] == '-' && argv[i][1] == 'o') {
			if (argv[i][2] == 0) {
				if (i + 1 == argc || (argv[i+1][0] == '-' && argv[i+1][1] != 0)) {
					fprintf(stderr, "ERROR: missing filename after \"-%c\"\n", argv[i][1]);
					goto exit;
				}
				out_file = argv[i+1];
				i++;
			} else {
				out_file = argv[i] + 2;
			}
		} else if (strcmp(argv[i], "--emit-c") == 0) {
			dump |= IR_DUMP_C;
		} else if (strcmp(argv[i], "--emit-llvm") == 0) {
			dump |= IR_DUMP_LLVM;
		} else if (strcmp(argv[i], "--save") == 0) {
			// TODO: check save/dot/dump/... conflicts
			dump |= IR_DUMP_IR;
			if (i + 1 < argc && argv[i + 1][0] != '-') {
				dump_file = argv[i + 1];
				i++;
			}
		} else if (strcmp(argv[i], "--emit-ir") == 0) {
			// TODO: check save/dot/dump/... conflicts
			dump |= IR_DUMP_IR;
			save_flags |= IR_SAVE_CFG | IR_SAVE_SAFE_NAMES;
			if (i + 1 < argc && argv[i + 1][0] != '-') {
				dump_file = argv[i + 1];
				i++;
			}
		} else if (strcmp(argv[i], "--save-cfg") == 0) {
			save_flags |= IR_SAVE_CFG;
		} else if (strcmp(argv[i], "--save-cfg-map") == 0) {
			save_flags |= IR_SAVE_CFG | IR_SAVE_CFG_MAP;
		} else if (strcmp(argv[i], "--save-rules") == 0) {
			save_flags |= IR_SAVE_RULES;
		} else if (strcmp(argv[i], "--save-regs") == 0) {
			save_flags |= IR_SAVE_REGS;
		} else if (strcmp(argv[i], "--save-use-lists") == 0) {
			save_flags |= IR_SAVE_USE_LISTS;
		} else if (strcmp(argv[i], "--save-dot") == 0) {
			dump |= IR_DUMP_DOT;
			if (i + 1 < argc && argv[i + 1][0] != '-') {
				dump_file = argv[i + 1];
				i++;
			}
		} else if (strcmp(argv[i], "--save-live-ranges") == 0) {
			dump |= IR_DUMP_LIVE_RANGES;
		} else if (strcmp(argv[i], "--save-ir-codegen") == 0) {
			dump |= IR_DUMP_CODEGEN;
		} else if (strcmp(argv[i], "--save-ir-after-load") == 0) {
			dump |= IR_DUMP_AFTER_LOAD;
		} else if (strcmp(argv[i], "--save-ir-after-use-lists") == 0) {
			dump |= IR_DUMP_AFTER_USE_LISTS;
		} else if (strcmp(argv[i], "--save-ir-after-mem2ssa") == 0) {
			dump |= IR_DUMP_AFTER_MEM2SSA;
		} else if (strcmp(argv[i], "--save-ir-after-sccp") == 0) {
			dump |= IR_DUMP_AFTER_SCCP;
		} else if (strcmp(argv[i], "--save-ir-after-cfg") == 0) {
			dump |= IR_DUMP_AFTER_CFG;
		} else if (strcmp(argv[i], "--save-ir-after-dom") == 0) {
			dump |= IR_DUMP_AFTER_DOM;
		} else if (strcmp(argv[i], "--save-ir-after-loop") == 0) {
			dump |= IR_DUMP_AFTER_LOOP;
		} else if (strcmp(argv[i], "--save-ir-after-gcm") == 0) {
			dump |= IR_DUMP_AFTER_GCM;
		} else if (strcmp(argv[i], "--save-ir-after-scheduling") == 0) {
			dump |= IR_DUMP_AFTER_SCHEDULING;
		} else if (strcmp(argv[i], "--save-ir-after-matching") == 0) {
			dump |= IR_DUMP_AFTER_CODE_MATCHING;
		} else if (strcmp(argv[i], "--save-ir-after-live-ranges") == 0) {
			dump |= IR_DUMP_AFTER_LIVE_RANGES;
		} else if (strcmp(argv[i], "--save-ir-after-coalescing") == 0) {
			dump |= IR_DUMP_AFTER_COALESCING;
		} else if (strcmp(argv[i], "--save-ir-after-regalloc") == 0) {
			dump |= IR_DUMP_AFTER_REGALLOC;
		} else if (strcmp(argv[i], "--save-ir-after-each-pass") == 0) {
			dump |= IR_DUMP_AFTER_ALL;
		} else if (strcmp(argv[i], "--save-ir-final") == 0) {
			dump |= IR_DUMP_FINAL;
		} else if (strcmp(argv[i], "--dump-size") == 0) {
			dump |= IR_DUMP_SIZE;
		} else if (strcmp(argv[i], "--dump-time") == 0) {
			dump_time = 1;
		} else if (strcmp(argv[i], "-S") == 0) {
			dump |= IR_DUMP_ASM;
		} else if (strcmp(argv[i], "--run") == 0) {
			dump |= IR_RUN;
			if (i + 1 < argc) {
				run_args = i + 1;
			}
			break;
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
		} else if (strcmp(argv[i], "-mavx") == 0) {
			mflags |= IR_X86_AVX;
		} else if (strcmp(argv[i], "-mbmi1") == 0) {
			mflags |= IR_X86_BMI1;
			mflags_disabled &= ~IR_X86_BMI1;
		} else if (strcmp(argv[i], "-mno-bmi1") == 0) {
			mflags &= ~IR_X86_BMI1;
			mflags_disabled |= IR_X86_BMI1;
		} else if (strcmp(argv[i], "-msse3") == 0) {
			mflags |= IR_X86_SSE3;
			mflags_disabled &= ~IR_X86_SSE3;
		} else if (strcmp(argv[i], "-mno-sse3") == 0) {
			mflags &= ~IR_X86_SSE3;
			mflags_disabled |= IR_X86_SSE3;
		} else if (strcmp(argv[i], "-mssse3") == 0) {
			mflags |= IR_X86_SSSE3;
			mflags_disabled &= ~IR_X86_SSSE3;
		} else if (strcmp(argv[i], "-mno-ssse3") == 0) {
			mflags &= ~IR_X86_SSSE3;
			mflags_disabled |= IR_X86_SSSE3;
		} else if (strcmp(argv[i], "-msse4") == 0) {
			mflags |= IR_X86_SSE41 | IR_X86_SSE42;
			mflags_disabled &= ~(IR_X86_SSE41 | IR_X86_SSE42);
		} else if (strcmp(argv[i], "-mno-sse4") == 0) {
			mflags &= ~(IR_X86_SSE41 | IR_X86_SSE42);
			mflags_disabled |= IR_X86_SSE41 | IR_X86_SSE42;
		} else if (strcmp(argv[i], "-msse4.1") == 0) {
			mflags |= IR_X86_SSE41;
			mflags_disabled &= ~IR_X86_SSE41;
		} else if (strcmp(argv[i], "-mno-sse4.1") == 0) {
			mflags &= ~IR_X86_SSE41;
			mflags_disabled |= IR_X86_SSE41;
		} else if (strcmp(argv[i], "-msse4.2") == 0) {
			mflags |= IR_X86_SSE42;
			mflags_disabled &= ~IR_X86_SSE42;
		} else if (strcmp(argv[i], "-mno-sse4.2") == 0) {
			mflags &= ~IR_X86_SSE42;
			mflags_disabled |= IR_X86_SSE42;
		} else if (strcmp(argv[i], "-mavx2") == 0) {
			mflags |= IR_X86_AVX2;
			mflags_disabled &= ~IR_X86_AVX2;
		} else if (strcmp(argv[i], "-mno-avx2") == 0) {
			mflags &= ~IR_X86_AVX2;
			mflags_disabled |= IR_X86_AVX2;
#endif
		} else if (strcmp(argv[i], "-muse-fp") == 0) {
			flags |= IR_USE_FRAME_POINTER;
		} else if (strcmp(argv[i], "-mfastcall") == 0) {
			flags |= IR_CC_FASTCALL;
		} else if (strcmp(argv[i], "-fno-inline") == 0) {
			disable_inline = 1;
		} else if (strcmp(argv[i], "-finline") == 0) {
			force_inline = 1;
		} else if (strcmp(argv[i], "-fno-tail-calls") == 0) {
			disable_tail_calls = 1;
		} else if (strcmp(argv[i], "-ftail-calls") == 0) {
			force_tail_calls = 1;
		} else if (strcmp(argv[i], "-fno-mem2ssa") == 0) {
			disable_mem2ssa = 1;
#ifdef IR_DEBUG
		} else if (strcmp(argv[i], "--debug-sccp") == 0) {
			flags |= IR_DEBUG_SCCP;
		} else if (strcmp(argv[i], "--debug-gcm") == 0) {
			flags |= IR_DEBUG_GCM;
		} else if (strcmp(argv[i], "--debug-gcm-split") == 0) {
			flags |= IR_DEBUG_GCM_SPLIT;
		} else if (strcmp(argv[i], "--debug-scheduling") == 0) {
			flags |= IR_DEBUG_SCHEDULE;
		} else if (strcmp(argv[i], "--debug-regalloc") == 0) {
			flags |= IR_DEBUG_RA;
		} else if (strcmp(argv[i], "--debug-bb-scheduling") == 0) {
			flags |= IR_DEBUG_BB_SCHEDULE;
#endif
		} else if (strcmp(argv[i], "--debug-regset") == 0) {
			if (i + 1 == argc || argv[i + 1][0] == '-') {
				fprintf(stderr, "ERROR: Invalid usage' (use --help)\n");
				return 1;
			}
			debug_regset = strtoull(argv[i + 1], NULL, 0);
			i++;
#ifdef _WIN32
		} else if (strcmp(argv[i], "--no-abort-fault") == 0) {
			abort_fault = 0;
#endif
#if HAVE_LLVM
		} else if (strcmp(argv[i], "--llvm-bitcode") == 0) {
			if (input || i + 1 == argc || argv[i + 1][0] == '-') {
				fprintf(stderr, "ERROR: Invalid usage' (use --help)\n");
				return 1;
			}
			load_llvm_bitcode = 1;
			input = argv[++i];
		} else if (strcmp(argv[i], "--llvm-asm") == 0) {
			if (input || i + 1 == argc || argv[i + 1][0] == '-') {
				fprintf(stderr, "ERROR: Invalid usage' (use --help)\n");
				return 1;
			}
			load_llvm_asm = 1;
			input = argv[++i];
#endif
		} else if (strcmp(argv[i], "-g") == 0) {
			dump |= IR_GDB;
		} else if (strcmp(argv[i], "-p") == 0) {
			dump |= IR_PERF;
		} else if (strcmp(argv[i], "-fsyntax-only") == 0) {
			dump |= IR_LOAD_ONLY;
		} else if (argv[i][0] == '-') {
			fprintf(stderr, "ERROR: Unknown option '%s' (use --help)\n", argv[i]);
			return 1;
		} else {
			if (input) {
				fprintf(stderr, "ERROR: Invalid usage' (use --help)\n");
				return 1;
			}
			input = argv[i];
		}
	}

	if ((dump & (IR_DUMP_IR|IR_DUMP_LIVE_RANGES)) && !(dump & (IR_DUMP_AFTER_ALL|IR_DUMP_CODEGEN))) {
		dump |= IR_DUMP_FINAL;
	}

	if (!input) {
		fprintf(stderr, "ERROR: no input file\n");
		return 1;
	}

#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
	uint32_t cpuinfo = ir_cpuinfo();

	if (!(cpuinfo & IR_X86_SSE2)) {
		fprintf(stderr, "ERROR: incompatible CPU (SSE2 is not supported)\n");
		return 1;
	}

	if ((mflags & IR_X86_AVX) && !(cpuinfo & IR_X86_AVX)) {
		fprintf(stderr, "ERROR: -mavx is not compatible with CPU (AVX is not supported)\n");
		return 1;
	}
	mflags |= (cpuinfo & (IR_X86_SSE3|IR_X86_SSSE3|IR_X86_SSE41|IR_X86_SSE42|IR_X86_AVX2|IR_X86_BMI1)) & ~mflags_disabled;
#endif

#ifdef _WIN32
	if (!abort_fault) {
		_set_abort_behavior(0, _WRITE_ABORT_MSG|_CALL_REPORTFAULT);
		SetErrorMode(SEM_FAILCRITICALERRORS|SEM_NOGPFAULTERRORBOX);
	}
#endif

	flags |= IR_FUNCTION;

	if (opt_level > 0) {
		flags |= IR_OPT_FOLDING | IR_OPT_CFG | IR_OPT_CODEGEN;
	}
	if ((opt_level > 1 && !disable_inline) || force_inline) {
		flags |= IR_OPT_INLINE;
	}
	if ((opt_level > 1 && !disable_tail_calls) || force_tail_calls) {
		flags |= IR_OPT_TAILCALL;
	}
	if (opt_level > 0 && !disable_mem2ssa) {
		flags |= IR_OPT_MEM2SSA;
	}
	if (dump & IR_GEN_CODE) {
		if ((dump & (IR_DUMP_C|IR_DUMP_LLVM)) == (IR_DUMP_C|IR_DUMP_LLVM)) {
			fprintf(stderr, "ERROR: --emit-c and --emit-llvm are incompatible\n");
			return 1;
		}
	}
	if (dump & IR_GEN_NATIVE) {
		if (dump & IR_GEN_CODE) {
			fprintf(stderr, "ERROR: --emit-c and --emit-llvm are incompatible with native code generator (-S, --dump-size, --run)\n");
			return 1;
		}
	}
	if (dump & IR_LOAD_ONLY) {
		if (dump & IR_GEN_NATIVE) {
			fprintf(stderr, "ERROR: -fsyntax-only is not compatible with native compilation flags (-S, --run, --dump-size)\n");
			return 1;
		}
		if (dump & IR_GEN_CODE) {
			fprintf(stderr, "ERROR: -fsyntax-only is not compatible with --emit-c and --emit-llvm\n");
			return 1;
		}
	}

	if (!(dump & IR_RUN)) {
		dump &= ~(IR_GDB|IR_PERF);
	}

	memset(&loader, 0, sizeof(loader));
	loader.loader.default_func_flags = flags;
	loader.loader.init_module        = NULL;
	loader.loader.external_sym_dcl   = ir_loader_external_sym_dcl;
	loader.loader.external_func_dcl  = ir_loader_external_func_dcl;
	loader.loader.forward_func_dcl   = ir_loader_forward_func_dcl;
	loader.loader.sym_dcl            = ir_loader_sym_dcl;
	loader.loader.sym_data           = ir_loader_sym_data;
	loader.loader.sym_data_str       = ir_loader_sym_data_str;
	loader.loader.sym_data_pad       = ir_loader_sym_data_pad;
	loader.loader.sym_data_ref       = ir_loader_sym_data_ref;
	loader.loader.sym_data_end       = ir_loader_sym_data_end;
	loader.loader.func_init          = ir_loader_func_init;
	loader.loader.func_process       = ir_loader_func_process;
	loader.loader.resolve_sym_name   = _ir_loader_resolve_sym_name;
	loader.loader.has_sym            = ir_loader_has_sym;
	loader.loader.add_sym            = ir_loader_add_sym;
	loader.loader.add_label          = ir_loader_add_sym;

	loader.opt_level = opt_level;
	loader.mflags = mflags;
	loader.debug_regset = debug_regset;
	loader.save_flags = save_flags;
	loader.dump = dump;

	ir_strtab_init(&loader.symtab, 16, 4096);
	loader.sym = NULL;
	loader.reloc = NULL;
	loader.sym_count = 0;
	loader.reloc_count = 0;

	if (dump_file) {
		loader.dump_file = fopen(dump_file, "w+");
		if (!loader.dump_file) {
			fprintf(stderr, "ERROR: Cannot create file '%s'\n", dump_file);
			ret = 1;
			goto exit;
		}
	} else {
		loader.dump_file = stderr;
	}
	if (dump & (IR_DUMP_C|IR_DUMP_LLVM|IR_DUMP_ASM)) {
		if (!out_file) {
			loader.out_file = stderr;
		} else if (strcmp(out_file, "-") == 0) {
			loader.out_file = stdout;
		} else {
			loader.out_file = fopen(out_file, "w+");
			if (!loader.out_file) {
				fprintf(stderr, "ERROR: Cannot create file '%s'\n", out_file);
				ret = 1;
				goto exit;
			}
		}
	}

	if (dump & (IR_DUMP_ASM|IR_DUMP_SIZE|IR_RUN)) {
		/* Preallocate 2MB JIT code buffer. It may be necessary to generate veneers and thunks. */
		size_t size = 2 * 1024 * 1024;
		void *entry;

		loader.code_buffer.start = ir_mem_mmap(size);
		if (!loader.code_buffer.start) {
			fprintf(stderr, "ERROR: Cannot allocate JIT code buffer\n");
			ret = 1;
			goto exit;
		}
		loader.code_buffer.pos = loader.code_buffer.start;
		loader.code_buffer.end = (char*)loader.code_buffer.start + size;

		if (ir_needs_thunk(&loader.code_buffer, printf)) {
			ir_mem_unprotect(loader.code_buffer.start, (char*)loader.code_buffer.end - (char*)loader.code_buffer.start);
			entry = ir_emit_thunk(&loader.code_buffer, printf, &size);
			ir_loader_add_sym(&loader.loader, (void*)"printf", entry);
			ir_mem_protect(loader.code_buffer.start, (char*)loader.code_buffer.end - (char*)loader.code_buffer.start);
		} else {
			ir_loader_add_sym(&loader.loader, (void*)"printf", printf);
		}
		if (ir_needs_thunk(&loader.code_buffer, putchar)) {
			ir_mem_unprotect(loader.code_buffer.start, (char*)loader.code_buffer.end - (char*)loader.code_buffer.start);
			entry = ir_emit_thunk(&loader.code_buffer, putchar, &size);
			ir_loader_add_sym(&loader.loader, (void*)"putchar", entry);
			ir_mem_protect(loader.code_buffer.start, (char*)loader.code_buffer.end - (char*)loader.code_buffer.start);
		} else {
			ir_loader_add_sym(&loader.loader, (void*)"putchar", putchar);
		}
	}

#ifndef _WIN32
	if (dump & IR_PERF) {
		ir_perf_jitdump_open();
	}
#endif

	if (dump_time) {
		start = ir_time();
	}

#if HAVE_LLVM
	if (load_llvm_bitcode) {
		if (!ir_load_llvm_bitcode(&loader.loader, input)) {
			fprintf(stderr, "ERROR: Cannot load LLVM file '%s'\n", input);
			ret = 1;
			goto exit;
		}
		goto finish;
	} else if (load_llvm_asm) {
		if (!ir_load_llvm_asm(&loader.loader, input)) {
			fprintf(stderr, "ERROR: Cannot load LLVM file '%s'\n", input);
			ret = 1;
			goto exit;
		}
		goto finish;
	}
#endif

	f = fopen(input, "rb");
	if (!f) {
		fprintf(stderr, "ERROR: Cannot open input file '%s'\n", input);
		ret = 1;
		goto exit;
	}

	ir_loader_init();

	if (!ir_load(&loader.loader, f)) {
		fclose(f);
		ir_loader_free();
		ret = 2;
		goto exit;
	}

	fclose(f);
	ir_loader_free();

#if HAVE_LLVM
finish:
#endif

	if (loader.dump_file && loader.dump_file != stderr && loader.dump_file != stdout) {
		fclose(loader.dump_file);
	}
	if (loader.out_file && loader.out_file != stderr && loader.out_file != stdout) {
		fclose(loader.out_file);
	}

	if (!ir_loader_fix_relocs(&loader)) {
		if ((dump & IR_RUN) && loader.main) {
			fprintf(stderr, "ERROR: Cannot run program with undefined symbols\n");
			ret = 1;
			goto exit;
		}
	}

	if (dump & IR_DUMP_SIZE) {
		if (loader.code_buffer.start) {
			loader.size = (char*)loader.code_buffer.pos - (char*)loader.code_buffer.start;
		}
		fprintf(stderr, "\ncode size = %lld\n", (long long int)loader.size);
	}

	if (dump_time) {
		double t = ir_time();
		fprintf(stderr, "\ncompilation time = %0.6f\n", t - start);
		start = t;
	}

	if ((dump & IR_RUN) && loader.main) {
		int jit_argc = 1;
		const char **jit_argv;
		int (*func)(int, const char**) = loader.main;

		if (dump_time) {
			ir_atexit_start = start;
			atexit(ir_atexit);
		}

		if (run_args && argc > run_args) {
			jit_argc = argc - run_args + 1;
		}
		jit_argv = alloca(sizeof(char*) * jit_argc);
		jit_argv[0] = "jit code";
		for (i = 1; i < jit_argc; i++) {
			jit_argv[i] = argv[run_args + i - 1];
		}
#if defined(_MSC_VER) && defined(IR_TARGET_X86)
		ret = ir_call_with_aligned_stack(func, jit_argc, jit_argv);
#else
		ret = func(jit_argc, jit_argv);
#endif

		if (dump_time) {
			double t = ir_time();
			fprintf(stderr, "\nexecution time = %0.6f\n", t - start);
			ir_atexit_start = 0.0;
		}

#ifndef _WIN32
		if (dump & IR_PERF) {
			ir_perf_jitdump_close();
		}
#endif
	}

exit:
	ir_loader_free_symbols(&loader);
	return ret;
}