##******************************************************************************
##  Copyright(C) 2011-2012 Intel Corporation. All Rights Reserved.
##
##  The source code, information  and  material ("Material") contained herein is
##  owned  by Intel Corporation or its suppliers or licensors, and title to such
##  Material remains  with Intel Corporation  or its suppliers or licensors. The
##  Material  contains proprietary information  of  Intel or  its  suppliers and
##  licensors. The  Material is protected by worldwide copyright laws and treaty
##  provisions. No  part  of  the  Material  may  be  used,  copied, reproduced,
##  modified, published, uploaded, posted, transmitted, distributed or disclosed
##  in any way  without Intel's  prior  express written  permission. No  license
##  under  any patent, copyright  or  other intellectual property rights  in the
##  Material  is  granted  to  or  conferred  upon  you,  either  expressly,  by
##  implication, inducement,  estoppel or  otherwise.  Any  license  under  such
##  intellectual  property  rights must  be express  and  approved  by  Intel in
##  writing.
##
##  *Third Party trademarks are the property of their respective owners.
##
##  Unless otherwise  agreed  by Intel  in writing, you may not remove  or alter
##  this  notice or  any other notice embedded  in Materials by Intel or Intel's
##  suppliers or licensors in any way.
##
##******************************************************************************
##  Content:
##      Makefile for SGEMM, DGEMM, CGEMM, ZGEMM benchmarks
##******************************************************************************

ICC		?= icc
TARGET_ICC	?= $(ICC)
INSTALL		?= install
VERSION		?= 0.0.0
prefix		?= /usr
exec_prefix	?= $(prefix)
libexecdir	?= $(exec_prefix)/libexec
datarootdir	?= $(prefix)/share
docdir		?= $(datarootdir)/doc
srcinstalldir	?= $(prefix)/src/micperf
card_libs ?= /opt/mpss/x200/minsdk/lib

all: knc

COMPOSER_VER = $(if $(findstring 2018, $(MKLROOT)),2018,2017)
ifeq ($(COMPOSER_VER),2018)
addprec = $(foreach x, s d i, $(x)$(strip $(1)))
else
addprec = $(foreach x, s d, $(x)$(strip $(1)))
endif

CPU_TARGETS	= $(call addprec, gemm_cpu.x)
MIC_TARGETS	= $(call addprec, gemm_mic.x)
OFL_TARGETS	= $(call addprec, gemm_ofl.x)
MCDRAM_CPU_TARGETS = $(call addprec, gemm_mcdram_cpu.x)
KNL_SNC_TARGETS = $(call addprec, gemm_mpi_snc_cpu.x)

TARGETS		= $(MIC_TARGETS) $(CPU_TARGETS) $(OFL_TARGETS) \
$(MCDRAM_CPU_TARGETS) $(KNL_SNC_TARGETS)

SOURCE          = bench.c bench.h main.c Makefile README.txt redist.txt utils.c utils.h cross_timer.h

# BASIC COMPILATION SETUP
MKL_CFLAGS	= -I$(MKLROOT)/include -qopenmp

# SETTING DYNAMIC=yes at make time forces dynamic linking with MKL
ifneq (yes,$(DYNAMIC))
WLSTATIC	= -Wl,-Bstatic
WLDYNAMIC	= -Wl,-Bdynamic
STATIC		= -Bstatic
DYNAMIC		= -Bdynamic
endif
MKL_LIBS	= $(WLSTATIC) -lmkl_intel_lp64 -Wl,--start-group \
                  -lmkl_intel_thread -lmkl_core $(WLDYNAMIC) -Wl,--end-group
MKL_LIBS_LINKER	= $(STATIC) -lmkl_intel_lp64 --start-group \
                  -lmkl_intel_thread -lmkl_core $(DYNAMIC) --end-group

K1OM_BIN_POSTFIX = /mic

MIC_CFLAGS	= $(MKL_CFLAGS) -qno-offload

CPU_CFLAGS	= $(MKL_CFLAGS) -qno-offload -D_GNU_SOURCE
CPU_LIBS	= -L$(MKLROOT)/lib/intel64 $(MKL_LIBS)

knc: OFL_CFLAGS	= $(MKL_CFLAGS)
knc: OFL_LIBS	= $(CPU_LIBS) -offload-option,mic,ld,"-L$(MKLROOT)/lib/mic $(MKL_LIBS_LINKER) --no-undefined"
knc: MIC_LIBS	= -L$(MKLROOT)/lib/mic $(MKL_LIBS)

knllb: OFL_CFLAGS = $(MKL_CFLAGS) -qoffload-arch=mic-avx512 -qoffload-option,mic,compiler,"-L$(card_libs)"
knllb: OFL_LIBS = $(CPU_LIBS) -qoffload-option,mic,ld,"-L$(MKLROOT)/lib/intel64 $(MKL_LIBS_LINKER) -L$(card_libs) --no-undefined" -qoffload-option,mic,link,"-rpath-link=$(card_libs)"
knllb: MIC_LIBS	= -L$(MKLROOT)/lib/intel64 $(MKL_LIBS)

ALL_CFLAGS	= -O2 -Wall -Werror -g -std=c99 -D_FORTIFY_SOURCE=2 -fstack-protector
EXTRA_LDFLAGS = -Wl,-z,relro,-z,now

# PRECISION OPTIONS SELECTION
s%.x	: PREC	= -DSINGLE_PREC
d%.x	: PREC	= -DDOUBLE_PREC
c%.x	: PREC	= -DCOMPLEX
z%.x	: PREC	= -DCOMPLEX16
i%.x    : PREC  = -DINTEGER

# ------- MKL ----------
$(MIC_TARGETS)	: CC		= $(TARGET_ICC)
$(MIC_TARGETS)	: EXTRA_CFLAGS	= $(MIC_CFLAGS)
$(MIC_TARGETS)	: LIBS		= $(MIC_LIBS)
$(CPU_TARGETS)	: CC		= $(ICC)
$(CPU_TARGETS)	: EXTRA_CFLAGS	= $(CPU_CFLAGS)
$(CPU_TARGETS)	: LIBS		= $(CPU_LIBS)
$(OFL_TARGETS)	: CC		= $(ICC)
$(OFL_TARGETS)	: EXTRA_CFLAGS	= $(OFL_CFLAGS)
$(OFL_TARGETS)	: LIBS		= $(OFL_LIBS)
$(MCDRAM_CPU_TARGETS): CC        = $(ICC)
$(MCDRAM_CPU_TARGETS): EXTRA_CFLAGS  = $(CPU_CFLAGS) -DMCDRAM_KNLSB
$(MCDRAM_CPU_TARGETS): LIBS      = $(CPU_LIBS) -lmemkind
$(KNL_SNC_TARGETS): CC		= mpiicc
$(KNL_SNC_TARGETS): EXTRA_CFLAGS  = $(CPU_CFLAGS) -DKNL_SNC_MODE
$(KNL_SNC_TARGETS): LIBS      = $(CPU_LIBS) -lnuma

# Optimize KNL binaries to use AVX512 and KNC card binaries for the k1om architecture
knc: MIC_CFLAGS += -mmic
knlsb: CPU_CFLAGS += -axMIC-AVX512,CORE-AVX512,SSE4.2
knllb: MIC_CFLAGS += -axMIC-AVX512

knc: mkl-x86_64 mkl-k1om

knlsb: $(CPU_TARGETS) $(MCDRAM_CPU_TARGETS) $(KNL_SNC_TARGETS)

knllb: mkl-x86_64 mkl-k1om

install: install-knc

install-knc: install-x86_64 install-k1om install-license

install-knlsb: $(CPU_TARGETS) $(MCDRAM_CPU_TARGETS) $(KNL_SNC_TARGETS)
	$(INSTALL) -d $(DESTDIR)$(libexecdir)/micperf/x86_64_AVX512
	$(INSTALL) $^ $(DESTDIR)$(libexecdir)/micperf/x86_64_AVX512
	$(INSTALL) -d $(DESTDIR)$(docdir)

install-knllb: install-x86_64 install-x86_64-avx512 install-license

mkl-x86_64: $(OFL_TARGETS) $(CPU_TARGETS)

mkl-k1om: $(MIC_TARGETS)

install-x86_64: $(OFL_TARGETS) $(CPU_TARGETS)
	$(INSTALL) -d $(DESTDIR)$(libexecdir)/micperf/x86_64
	$(INSTALL) $^ $(DESTDIR)$(libexecdir)/micperf/x86_64

install-k1om: $(MIC_TARGETS)
	$(INSTALL) -d $(DESTDIR)$(libexecdir)/micperf/k1om
	$(INSTALL) $^ $(DESTDIR)$(libexecdir)/micperf/k1om

install-x86_64-avx512: $(MIC_TARGETS)
	$(INSTALL) -d $(DESTDIR)$(libexecdir)/micperf/x86_64_AVX512
	$(INSTALL) $^ $(DESTDIR)$(libexecdir)/micperf/x86_64_AVX512

install-license:
	$(INSTALL) -d $(DESTDIR)$(docdir)

clean:
	-rm -rf $(TARGETS)

$(TARGETS): main.c bench.c bench.h utils.c utils.h cross_timer.h

%.x:
	$(CC) $(ALL_CFLAGS) $(CPPFLAGS) $(EXTRA_CFLAGS) $(CFLAGS) $(PREC) $(LDFLAGS) $(EXTRA_LDFLAGS) $(filter %.c, $^) $(LIBS) -o $@

.PHONY: all knc knlsb knllb mkl-x86_64 mkl-k1om mkl-ofl mkl-cpu mkl-mic clean install install-knc install-knlsb install-knllb install-x86_64 install-k1om
