<?php
namespace App\Cron\Job;

/**
 * Gatherer for the BSP tickets entities
 * Used in App\Cron\Job\PopulateBspSettlementRecords
 * @author Lucian Sirbu
 *
 */
class PopulateBspSettlementRecords extends \App\Cron\Job\Utils\LongRunningCronJobAbstract
{
	/**
	 * Cronjob category: {system|operational|financial|reporting}
	 * @var string
	 */
	protected $category = 'financial';


	protected $defaultSettings = [
		'resetOnFirstRun' => 		false,
		'ignoredSuppliers'	=> 		[],
	];

	protected $queuedBspSettlements = [];

	/**
	 * Script/DB stats
	 * @var array
	 */
	protected $stats = [];

	/**
	 * The local bsp vs importedService record comparer
	 * @var \App\Bsp\Settlement\CompareWithImportedService
	 */
	private $recordComparer;

	public function __construct()
	{
		parent::__construct();

		$this->options['StopAndRestartJob']['maxRestarts'] = 24*60*60;	// 86400 restarts, or 1 day, if it retries every 1 minute
		$this->options['StopAndRestartJob']['maxDurationOfRestarts'] = 24*60*60;	// 1 day

		$this->options['bspSettlementsQueueLength'] = 10000;

		$this->stats['flushQueuedBspSettlements'] = 0;
		$this->stats['insertedRows'] = 0;
		$this->stats['updatedRows'] = 0;
		$this->stats['existingRows'] = 0;
	}

	/**
	 * (non-PHPdoc)
	 * @see \App\Cron\Job\Utils\GeneralAbstract::getSettingsDialog()
	 */
	public function getSettingsDialog(array $fields)
	{
		$form = parent::getSettingsDialog($fields);

		$_LANG = \App\Loader_Lang::getInstance()->getGlobal();

		$_LANG = \App\Loader_Lang::getInstance()->getGlobal();
		$lang = \App\Loader_Lang::getInstance()->getSection("Admin/CronJobs/Management/PopulateBspSettlementRecords");

		$formContainer = \Engine\Form\Container\Fieldset::create($_LANG['Admin']['CronJobs']['Management']['formFields']['legend']);

		$form->addContainer($formContainer);

		$fld = new \Engine\Form\Input\Checkbox($lang['options']['resetOnFirstRun'], 'resetOnFirstRun', $fields['settings']['resetOnFirstRun']);
		$fld->setTooltip($lang['options']['resetOnFirstRunTooltip']);
		$formContainer->addElement($fld);

		$fld = new \App\Form\Input\Cloned_AutocompleteSupplier($lang['options']['ignoredSuppliersList'], 'ignoredSuppliersList', $fields['settings']['ignoredSuppliersList']);
		$formContainer->addElement($fld);

		$formContainer->groupInputsInArray("settings");

		return $form;
	}

	/**
	 * {@inheritDoc}
	 * @see \App\Cron\Job\Utils\GeneralAbstract::cleanupSettings()
	 */
	public function cleanupSettings(array $saveData)
	{
		if ($saveData['settings']['ignoredSuppliersList']) {
			// convert the input-autocomplete format to a linear ID list
			$cleanedUpIdList = array_values(array_filter(array_unique(array_map(function($a) {
				return $a['id'];
			}, $saveData['settings']['ignoredSuppliersList']))));

			// format the id list back to an autocomplete-like structure, without any title
			$saveData['settings']['ignoredSuppliersList'] = array_map(function($id) {
				return ['id'=>$id];
			}, $cleanedUpIdList);
		}

		return $saveData;
	}

	public function getDefaultFormData()
	{
		$formData = [
			'options' => [
				'type' => 			'scheduled',
				'weekDays' => 		[],	// never runs
				'monthDays' => 		[], // never runs
				'timeInterval' => 	[
					'start' => 	'00:00:00',
					'end' => 	'23:59:59',
				],
				'executeTime' => 	'00:00:00',
				'repeatPeriod' => 	'01:00:00',
			],

			'settings' => $this->defaultSettings,
		];

		return $formData;
	}


	/**
	 * (non-PHPdoc)
	 * @see \App\Cron\Job\Utils\GeneralAbstract::executeJob()
	 */
	protected function executeJob()
	{
		if (\App\Cron\Main::getCronjobObj('PopulateBspPeriodCaching')->isAnotherInstanceRunning()) {
			throw new \Exception\CronJob\StopAndRestartJob('PopulateBspPeriodCaching is running. Retrying in 1 minute.');
			return;
		}
		if (\App\Cron\Main::getCronjobObj('PopulateBspPeriodCaching')->isQueued()) {
			throw new \Exception\CronJob\StopJob('PopulateBspPeriodCaching is scheduled to run. Aborting run.');
			return;
		}

		// act like a superuser
		\App\UserAccess\CurrentUserAccessFast::getInstance()->becomeSuperUser();

		// force that all those entity caches are stored in local memory
		\App\Entity\Cache::getInstance()->setCacheObject( new \Engine\Cache\LocalMemoryLinear() );

		// disable user trails. It led to out-of-memory on Fursan's server
		\App\UserTrail::getInstance()->storeTrail(false);

		if ($this->settings['resetOnFirstRun']) {
			$this->logger->info("resetOnFirstRun was set, reset BSP settlement data");

			$this->updateStatus([
				'currentStep' => 'resetting',
			]);

			// reset previous matches, if needed
			global $DBTBspSettlementRecords;
			$db = \DB::getInstance();
			$db->Execute("TRUNCATE TABLE `{$DBTBspSettlementRecords}`");
			if ($db->ErrorNo()) {
				throw new \DBException("Cannot select from table");
			}

			// toggle the checkbox back
			$this->settings['resetOnFirstRun'] = false;
			$this->setSettings($this->settings);

			$this->logger->success(sprintf("Cleared previous matches"));
		}

		$this->updateStatus([
			'progressSteps' => 	3,
		]);

		// STEP 1, pre-populate BSP records
		$startDate = microtime(true);
		$this->updateStatus([
			'progressStep' => 	0,
			'currentStep' => 	'importDataFromBspRecords',
		]);
		$this->importDataFromBspRecords();
		$this->flushQueuedBspSettlements();
		$this->logger->success(sprintf("Scanned all BSP records. It took %.1fs", microtime(true) - $startDate));

		// STEP 2, iterate over ImportedOrder services
		$startDate = microtime(true);
		$this->updateStatus([
			'progressStep' => 	1,
			'currentStep' => 	'importDataFromImportedServices',
		]);

		$this->importDataFromImportedServices();
		$this->flushQueuedBspSettlements();
		$this->logger->success(sprintf("Scanned all imported services. It took %.1fs", microtime(true) - $startDate));

		// STEP 3, iterate over BspSettlements and calculate matches
		$startDate = microtime(true);
		$this->updateStatus([
			'progressStep' => 	2,
			'currentStep' => 	'matchWithBspSettlements',
		]);
		$this->matchWithBspSettlements();
		$this->flushQueuedBspSettlements();
		$this->logger->success(sprintf("Scanned all ImportedOrder services. It took %.1fs", microtime(true) - $startDate));

		$this->logger->info(sprintf("All done"));
		$this->logger->info(sprintf("Inserted %s rows", number_format($this->stats['insertedRows'])));
		$this->logger->info(sprintf("Updated %s rows", number_format($this->stats['updatedRows'])));
		$this->logger->info(sprintf("Re-checked %s rows", number_format($this->stats['existingRows'])));

		$this->logger->info(sprintf("Done %d queue flushes, processed %.1f rps (estimated)",
			$this->stats['flushQueuedBspSettlements'],
			($this->stats['insertedRows'] + $this->stats['updatedRows'] + $this->stats['existingRows'])/(microtime(true)-$this->startDate)));

		$this->logger->info(sprintf("Queue cronjob ImportRefundsFromBsp to run"));
		\App\Cron\Main::getCronjobObj('ImportRefundsFromBsp')->addToQueue();

		\App\UserAccess\CurrentUserAccessFast::getInstance()->becomeNormalUser();
	}

	protected function getLastProcessedBspRecordId()
	{
		/*
		 * To determine the 'safe' ID from which we can resume, we look at the last inserted element in BspSettlement,
		 * 	and cross your fingers that the order of inserted elements in this table is incremental
		 */
		global $DBTBspSettlementRecords;
		$db = \DB::getInstance();
		$max = $db->getOne("SELECT MAX(`bspRecordId`) FROM `{$DBTBspSettlementRecords}`");
		if ($db->ErrorNo()) {
			throw new \DBException("Cannot select from table");
		}

		return $max;
	}

	protected function getLastProcessedImportedOrderServiceId()
	{
		/*
		 * To determine the 'safe' ID from which we can resume, we look at the last inserted element in BspSettlement, look at that importedService's addTime/updateTime,
		 * 	and then look in the importedServices list for other services with addTime/updateTime greater than the above determined date.
		 * 	the minimum ID found with these conditions should be considered safe to resume the scan, without any danger of skipping any service
		 */
		global $DBTBspSettlementRecords;
		$db = \DB::getInstance();
		$max = $db->getOne("SELECT MAX(`importedServiceId`) FROM `{$DBTBspSettlementRecords}`");
		if ($db->ErrorNo()) {
			throw new \DBException("Cannot select from table");
		}
		if ($max) {
			$importedService = \Entity\ImportedService::select($max);
			$lastRunDatetime = min(array_filter([DATE_LIMIT_MAX, $importedService->addTime, $importedService->lastUpdateTime, $importedService->blockTime, $importedService->processTime, ]));

			global $DBTImportedOrders_services;
			$safeIdToResume = $db->getOne("SELECT MIN(`id`) FROM `{$DBTImportedOrders_services}` WHERE `addTime`>=? OR lastUpdateTime>=? OR blockTime>=? OR processTime>=?", [$lastRunDatetime, $lastRunDatetime, $lastRunDatetime, $lastRunDatetime, ]);
			if ($db->ErrorNo()) {
				throw new \DBException("Cannot select from table");
			}

			return $safeIdToResume;

		}
		return 0;
	}


	/**
	 * Create table key from a BspRecord
	 * @param \App\Bsp\BspRow $bspTicket
	 * @return string
	 */
	protected function createKeyFromBsp(\App\Bsp\BspRow $bspTicket)
	{
		$amountStr = ($bspTicket->extraData->isRefund?'-':'+');

		// $ret = sprintf("%s/%s/%s", floor(\Engine\DateTime::createFromIsoDate($bspTicket->ticketDate)->getTimestamp()/(60*60*24*7)), $bspTicket->ticketNumber, $amountStr);
		// $ret = sprintf("%s/%s/%s", \Engine\DateTime::createFromIsoDate($bspTicket->ticketDate)->format("Ym"), $bspTicket->ticketNumber, $amountStr);

		if ($amountStr=='+') {	// issue
			$ret = sprintf("%s/%s/%s", \Engine\DateTime::createFromIsoDate($bspTicket->ticketDate)->format("Ym"), $bspTicket->ticketNumber, $amountStr);
		}
		else {	// refund
			// lookup the original ticket issue
			$bspIssue = null;

			// it might be in the internal queue
			$this->flushQueuedBspSettlements();
			if (!$bspIssue) {
				$ymRefund = (int)\Engine\DateTime::createFromIsoDate($bspTicket->ticketDate)->format("Ym");
				$ymRange = range($ymRefund, $ymRefund-12, -1);
				foreach ($ymRange as $ym) {
					$issueKey = sprintf("%s/%s/%s", $ym, $bspTicket->ticketNumber, '+');
					if (isset($this->queuedBspSettlements[$issueKey])) {
						$bspIssue = \Entity\BspRecord::select($this->queuedBspSettlements[$issueKey]->getField('bspRecordId'))->data;
						break;
					}
				}
			}


			// in might be in the DB
			if (!$bspIssue) {
				$bspRecords = $this->searchBspRecords(	// select initial sale
					$bspTicket->ticketNumber,
					$bspTicket->supplierCode3,
					\Engine\DateTime::createFromIsoDate($bspTicket->ticketDate)->modify("-12months")->formatDateIso(),
					true
				);

				if ($bspRecords) {
					$bspRecordId = end($bspRecords);
					$bspIssue = \Entity\BspRecord::select($bspRecordId)->data;
				}
			}

			// nowhere to be found. just create the key from the available RFND data
			if (!$bspIssue) {
				$bspIssue = $bspTicket;
			}

			$ret = sprintf("%s/%s/%s", \Engine\DateTime::createFromIsoDate($bspIssue->ticketDate)->format("Ym"), $bspIssue->ticketNumber, $amountStr);
		}
		return $ret;
	}

	/**
	 * Create table key from a ImportedOrder issued service
	 * @param array $importedService
	 * @return string
	 */
	protected function createKeyFromImportedServiceForIssue($importedService)
	{
		$amountStr = '+';

		// $ret = sprintf("%s/%s/%s", floor(\Engine\DateTime::createFromIsoDate($importedService['ticketDate'])->getTimestamp()/(60*60*24*7)), $importedService['ticketNumber'], $amountStr);
		// $ret = sprintf("%s/%s/%s", \Engine\DateTime::createFromIsoDate($importedService['ticketDate'])->format("Ym"), $importedService['ticketNumber'], $amountStr);
		$ret = sprintf("%s/%s/%s", \Engine\DateTime::createFromIsoDate($importedService['ticketDate'])->format("Ym"), $importedService['ticketNumber'], $amountStr);
		return $ret;
	}

	/**
	 * Create table key from a ImportedOrder refunded service
	 * @param array $importedService
	 * @return string
	 */
	protected function createKeyFromImportedServiceForRefund($importedService)
	{
		$rebuiltRefund = \App\Utils\BaseParser\Utils\RebuildImportedOrderRefund::getInstance()->rebuildOriginalRefund(\Entity\ImportedService::select($importedService['idImportedService']));

		$amountStr = '-';

		$ret = [];
		$ret['ticketDate'] = 		$rebuiltRefund['date'];
		$ret['ticketNumber'] = 		$rebuiltRefund['ticketNumber'];
		$ret['importedServiceId'] = $importedService['idImportedService'];
		$ret['key'] = 				sprintf("%s/%s/%s", \Engine\DateTime::createFromIsoDate($ret['ticketDate'])->format("Ym"), $ret['ticketNumber'], $amountStr);

		if (!\Entity\BspSettlementRecord::select($ret['key'])->recordExists()) {
			/**
			 * we're trying to match a refund that only exists in IO, not in BSP.
			 * 	For compatibility reasons, we should create the key from the refund date, as it was originally done.
			 * @see tests for TD-5885 We have to display in BSP Settlements, in ticketdate files, for refunds the date of the refund operation/
			 * Not sure this is a critical compatibility to keep though, as these tickets do not have the corresponding issue in BSP, so they will end up being partially handled anyway
			 * */
			$refundDetails = \Entity\ImportedService::select($importedService['idImportedService'])->object->refundDetails;
			$ret['ticketDate'] = 		$refundDetails->date;
			$ret['key'] = 				sprintf("%s/%s/%s", \Engine\DateTime::createFromIsoDate($ret['ticketDate'])->format("Ym"), $ret['ticketNumber'], $amountStr);
		}

		return $ret;
	}

	/**
	 * Add a record to the BSP settlements queue
	 * @param array $row
	 */
	protected function queueBspSettlement($row)
	{
		// first, lookup internal queue, flush if needed, to allow the below 'select' to return fresh data from DB
		if ($this->queuedBspSettlements[$row['key']]) {
			// 'micro-flush': commit only the affected/needed row and remove it from the current queue
			// if we use this, then getLastProcessedBspRecordId() & getLastProcessedImportedOrderServiceId() should behave differently
			/*
			$this->queuedBspSettlements[$row['key']]->save();
			unset($this->queuedBspSettlements[$row['key']]);
			*/

			$this->flushQueuedBspSettlements();
		}

		$existingRecord = \Entity\BspSettlementRecord::select($row['key']);

		$updated = null;
		// if row NOT in DB, insert now
		// if row in DB, update bspRecordId if set, update status if non-empty, update ticketNumber & friends if any is set
		if ($existingRecord->key) {	// use `key` instead of recordExists() to allow the internal queue 'hack'
			// record exists in the DB, UPDATE it
			$newRecord = \Entity\BspSettlementRecord::update($row['key']);
			$this->stats['existingRows']++;
		}
		else {
			// record doesn't exist in the DB, INSERT it now
			$newRecord = \Entity\BspSettlementRecord::insert();

			$newRecord->key = $row['key'];
			$updated.= 'key,';

			$this->stats['insertedRows']++;
		}


		// if ($row['bspRecordId'] && $existingRecord->bspRecordId!=$row['bspRecordId']) {	// update the existing tickets with "new" data
		if ($row['bspRecordId'] && !$existingRecord->bspRecordId) {		// ignore "new" data
			$newRecord->bspRecordId = $row['bspRecordId'];
			$updated.= 'bspRecordId,';
		}
		if ($row['bspRecordId'] && !$row['importedServiceId'] && $row['importedServiceIsRefund']) {
			$newRecord->importedServiceIsRefund = $row['importedServiceIsRefund'];
		}

		// if ($row['importedServiceId'] && ($existingRecord->importedServiceId!=$row['importedServiceId'] || $existingRecord->importedServiceIsRefund!=$row['importedServiceIsRefund'])) {	// update the existing tickets with "new" data
		if ($row['importedServiceId'] && !$existingRecord->importedServiceId) {	// ignore "new" data
			$newRecord->importedServiceId = $row['importedServiceId'];
			$newRecord->importedServiceIsRefund = $row['importedServiceIsRefund'];
			$updated.= 'importedServiceId,';
		}

		if ($row['status'] && $existingRecord->status!=$row['status']) {
			$newRecord->status = $row['status'];
			$updated.= 'status,';
		}

		if ($row['ticketStatus'] && $existingRecord->ticketStatus!=$row['ticketStatus']) {
			$newRecord->ticketStatus = $row['ticketStatus'];
			$updated.= 'ticketStatus,';
		}

		if ($row['ticketNumber'] && $existingRecord->ticketNumber!=$row['ticketNumber']) {
			$newRecord->ticketNumber = $row['ticketNumber'];
			$updated.= 'ticketNumber,';
		}
		if ($row['ticketDate'] && $existingRecord->ticketDate!=$row['ticketDate']) {
			$newRecord->ticketDate = $row['ticketDate'];
			$updated.= 'ticketDate,';
		}
		if ($row['idOffice'] && $existingRecord->idOffice!=$row['idOffice']) {
			$newRecord->idOffice = $row['idOffice'];
			$updated.= 'idOffice,';
		}

		if (array_key_exists('comparisonInProgress', $row)) {
			$newRecord->comparisonInProgress = $row['comparisonInProgress'];
		}

		if ($updated) {
			$this->stats['updatedRows']++;
 			// add this row to the internal queue
			$this->queuedBspSettlements[$row['key']] = $newRecord;
		}

		// is it time to flush?
		if (count($this->queuedBspSettlements)>$this->options['bspSettlementsQueueLength']) {
			$this->flushQueuedBspSettlements();
		}
	}

	/**
	 * Flush the internal BSP settlements queue
	 */
	protected function flushQueuedBspSettlements()
	{
		\DB::startTransaction();
		foreach ($this->queuedBspSettlements as $entity) {
			$entity->save();
		}
		\DB::commitTransaction();
		$this->queuedBspSettlements = [];

		$this->stats['flushQueuedBspSettlements']++;
	}

	/**
	 * Step 1 in the cronjob's execution
	 * @throws \DBException
	 */
	protected function importDataFromBspRecords()
	{
		$db = \DB::getInstance();

		global $DBTBspRecords;
		$totalBspRecords = $db->getOne("SELECT COUNT(*) FROM `{$DBTBspRecords}`");
		if ($db->ErrorNo()) {
			throw new \DBException("Cannot select from table");
		}
		$this->logger->info(sprintf("Found %s BSP records", number_format($totalBspRecords)));
		if (!$totalBspRecords) {
			$this->logger->warning(sprintf("No pre-parsed BSP records found. The reported progress will not be accurate"));
			$totalBspRecords = 1;	// avoid division-by-zeo
		}

		$bspServicesIterator = new \ModulesAbstract\PluginReports\Report\Model\Gatherer\BspPeriodRowsCachedAndPaginated();
		$bspServicesIterator->setOption('doUnserialize', false);
		$bspServicesIterator->setOption('disableAutoPopulate', true);
		$bspServicesIterator->setOption('allowPartialCommits', true);

		$prevBspPeriod = null;
		$bspServicesIterator->setOnPopulateBspRecordsCallback(function($bspRow) use (&$prevBspPeriod) {
			if ($prevBspPeriod !== $bspRow['bspPeriodId']) {
				$this->updateStatus([
					'status' => sprintf('importDataFromBspRecords,populateBspRecords(period:%s)', $bspRow['bspPeriodId']),
				]);
			}
			$prevBspPeriod = $bspRow['bspPeriodId'];
		});

		$this->updateStatus([
			'status' => sprintf('importDataFromBspRecords,initialize'),
		]);

		$maxProcessedBspRecordId = $this->getLastProcessedBspRecordId();
		$this->logger->info(sprintf("Resume from bsp record %d", $maxProcessedBspRecordId));

		$bspServicesIterator->addFilter('bspRecordId:start', $maxProcessedBspRecordId);

		$index = 0;
		$bspServicesIterator->setPreFilterCallback(function($bspTicket) use (&$index, $totalBspRecords, $maxProcessedBspRecordId) {
			if ($bspTicket['id']<$maxProcessedBspRecordId) {
				$index++;
				$this->updateStatus([
					'progress' => min(1.0, $index/$totalBspRecords),
					'status' => sprintf('importDataFromBspRecords,resume(BSP id:%s)', $bspTicket['id']),
				]);

				return 'intern:skip';
			}
		});

		while ($bspTicket = $bspServicesIterator->next()) {
			$index++;

			$this->updateStatus([
				'progress' => min(1.0, $index/$totalBspRecords),
				'status' => sprintf('importDataFromBspRecords,process(BSP id:%s)', $bspTicket['id']),
			]);

			$bspRecord = \Entity\BspRecord::select($bspTicket['id'])->data;

			$settlement = [
				'key' => 					$this->createKeyFromBsp($bspRecord),
				'bspRecordId' => 			$bspTicket['id'],

				'idOffice' => 				$bspTicket['idOffice'],
				'ticketNumber' => 			$bspRecord->ticketNumber,
				'ticketDate' => 			$bspRecord->ticketDate,
				'addTime' => 				DATETIME,

				'comparisonInProgress' => 	1,
			];

			if ($bspRecord->extraData->isRefund) {
				$settlement['importedServiceIsRefund'] = 1;
			}
			$this->queueBspSettlement($settlement);
		}
	}

	/**
	 * Step 2 in the cronjob's execution
	 * @throws \DBException
	 * @throws \Exception
	 */
	protected function importDataFromImportedServices()
	{
		$db = \DB::getInstance();

		global $DBTImportedOrders_services;
		$totalImportedOrdersServices = $db->getOne("SELECT COUNT(*) FROM `{$DBTImportedOrders_services}`");
		if ($db->ErrorNo()) {
			throw new \DBException("Cannot select from table");
		}
		$this->logger->info(sprintf("Found %s ImportedOrders services", number_format($totalImportedOrdersServices)));
		if (!$totalImportedOrdersServices) {
			$this->logger->warning(sprintf("No pre-parsed importedServices records found. The reported progress will not be accurate"));
			$totalImportedOrdersServices = 1;	// avoid division-by-zeo
		}

		$systems = (\App\Bsp\Config::getInstance()->get("systems")?(array)unserialize(\App\Bsp\Config::getInstance()->get("systems")):[]);
		if (!$systems) {
			throw new \Exception\CronJob\StopJob('You did not select any systems to be scanned. There is nothing to do');
		}

		$associatedServiceCodes = (\App\Bsp\Config::getInstance()->get("services")?(array)unserialize(\App\Bsp\Config::getInstance()->get("services")):[]);
		if (!$associatedServiceCodes) {
			throw new \Exception\CronJob\StopJob('You did not select any services to be scanned. There is nothing to do');
		}

		$importedServicesIterator = new \ModulesAbstract\PluginReports\Report\Model\Gatherer\ImportedServices();
		$importedServicesIterator->setOption('selectedColumns:idSystem', true);
		$importedServicesIterator->setOption('selectedColumns:isBlocked', true);
		$importedServicesIterator->setOption('queueLength', 25000);	// default is 2500
		$importedServicesIterator->addFilter('systems', $systems);
		$importedServicesIterator->addFilter('services', $associatedServiceCodes);

		$this->logger->info(sprintf("  > scan systems: %s", implode(", ", $systems)));
		$this->logger->info(sprintf("  > scan services: %s", implode(", ", $associatedServiceCodes)));

		$this->updateStatus([
			'status' => sprintf('importDataFromImportedServices'),
		]);

		$maxImportedOrderServiceId = $this->getLastProcessedImportedOrderServiceId();
		$this->logger->info(sprintf("Resume from imported service %d", $maxImportedOrderServiceId));

		$importedServicesIterator->addFilter('importedServiceId:start', $maxImportedOrderServiceId);
		$importedServicesIterator->addFilter('importedServiceId:notInBspSettlementList', true);

		$index = 0;
		$ignoredServices = 0;
		$importedServicesIterator->setPreFilterCallback(function($importedService) use (&$index, $totalImportedOrdersServices, $maxImportedOrderServiceId) {
			if ($importedService['idImportedService']<$maxImportedOrderServiceId) {
				$index++;
				$this->updateStatus([
					'progress' => min(1.0, $index/$totalImportedOrdersServices),
					'status' => sprintf('importDataFromImportedServices,resuming(IO service:%s)', $importedService['idImportedService']),
				]);
				return 'intern:skip';
			}
		});

		$ignoredSuppliersList = array_filter(array_map(function($a) {
			return $a['id'];
		}, (array)$this->settings['ignoredSuppliersList']));
		$this->logger->info(sprintf(" > ignoring %d suppliers", count($ignoredSuppliersList)));

		// first, get the manual system id, we will need it later
		$manualIdSystem = \Entity\ExtResSystem::getSystemIdByCode('manual');

		while ($importedService = $importedServicesIterator->next()) {
			$index++;

			if ($importedService['isConjunction']) {
				continue;	// skip conjunctions, only match real tickets
			}

			$this->updateStatus([
				'progress' => min(1.0, $index/$totalImportedOrdersServices),
				'status' => sprintf('importDataFromImportedServices,process(IO service:%s)', $importedService['idImportedService']),
			]);

			if (!$importedService) {
				throw new \Exception("ImportedOrder service not found. Please investigate.");
			}

			// the user requested that we ignore this service
			if ($ignoredSuppliersList && in_array($importedService['idOfferSupplier'], $ignoredSuppliersList)) {
				$ignoredServices++;
				continue;
			}

			// ignore blocked manual services, this should fix an issue for SA/Fursan, where they imported multiple times the same service, and blocked the old ones
			if ($manualIdSystem && $importedService['idSystem']==$manualIdSystem && $importedService['isBlocked']) {
				$ignoredServices++;
				continue;
			}

			$ret = [
				'statuses' => [], 			// matched, unmatched, diff:price, diff:supplier
				'section' => null, 			// issue, refund
				'importedService' => null,
				'bspRecord' => null,
			];

			$key = null;
			if ($importedService['section']=='refund') {
				// handle RFND operations
				$key = $this->createKeyFromImportedServiceForRefund($importedService)['key'];

				$bspSettlementRecord = \Entity\BspSettlementRecord::select($key);
				if ($bspSettlementRecord->recordExists() && $bspSettlementRecord->status!=['unmatched']) {
					// optimization: skip this row, we've already matched it before
					// TODO: can we pre-skip these? Like, skip them from SQL
					//		- for this, we should SELECT from IO WHERE `id` NOT IN (SELECT importedServiceId FROM bspSettlementRecords WHERE `status`=unmatched)
					continue;
				}

				$bspRecord = \Entity\BspRecord::select($bspSettlementRecord->bspRecordId);

				if ($bspRecord->recordExists() && $importedService['ticketNumber']!=$bspRecord->ticketNumber) {
					/**
					 * $bspRecord comes from the matched ticket that was instead created by createKeyFromImportedServiceForRefund() which uses rebuildOriginalRefund() in the end
					 * This means that in some cases, we might request ticket 1111 but the rebuildOriginalRefund() call might decide to return 1111->2222.
					 * 	We need to skip the 2222 ticket, for now at least, because while iterating through all tickets, we will eventually reach 2222 also
					 *  */
					continue;
				}

				// fill in importedService data
				$refundDate = $importedService['importedService']->refundDetails->date;
				$importedService['operationDate'] = $refundDate;
			}
			elseif ($importedService['section']=='issue') {
				// handle TKT/REIS operations
				$key = $this->createKeyFromImportedServiceForIssue($importedService);

				$bspSettlementRecord = \Entity\BspSettlementRecord::select($key);
				if ($bspSettlementRecord->recordExists() && $bspSettlementRecord->status!=['unmatched']) {
					// optimization: skip this row, we've already matched it before
					// TODO: can we pre-skip these? Like, skip them from SQL
					//		- for this, we should SELECT from IO WHERE `id` NOT IN (SELECT importedServiceId FROM bspSettlementRecords WHERE `status`=unmatched)
					continue;
				}

				// fill in importedService data
				$bspRecord = \Entity\BspRecord::select($bspSettlementRecord->bspRecordId);
				$importedService['operationDate'] = $importedService['ticketDate'];
			}
			else {
				throw new \Exception("Unknown importedService section to report: {$importedService['section']}");
			}


			$ret['section'] = 			$importedService['section'];
			$ret['importedService'] = 	$importedService;
			$ret['bspRecord'] = 		$bspRecord;

			if ($bspRecord->recordExists()) {
				$this->queueBspSettlement([
					'key' => 					$key,
					'bspRecordId' => 			$bspRecord->getId(),
					'importedServiceId' => 		$importedService['idImportedService'],
					'importedServiceIsRefund' =>(int)($importedService['section']=='refund'),
					'ticketNumber' =>			$importedService['ticketNumber'],
					'ticketDate' => 			$importedService['operationDate'],
					'idOffice' => 				$importedService['idOffice'],

					'comparisonInProgress' => 1,
				]);
			}
			else {
				$this->queueBspSettlement([
					'key' => 					$key,
					'importedServiceId' => 		$importedService['idImportedService'],
					'importedServiceIsRefund' =>(int)($importedService['section']=='refund'),
					'ticketNumber' =>			$importedService['ticketNumber'],
					'ticketDate' => 			$importedService['operationDate'],
					'idOffice' => 				$importedService['idOffice'],

					'comparisonInProgress' => 1,
				]);
			}
		}

		$this->logger->info(sprintf("Scanned %d imported services, and ignored %d services", $index, $ignoredServices));
	}

	/**
	 * Step 3 in the cronjob's execution
	 * @throws \DBException
	 * @throws \Exception
	 */
	protected function matchWithBspSettlements()
	{
		global $DBTBspSettlementRecords;
		$db = \DB::getInstance();
		$db->Execute("UPDATE `{$DBTBspSettlementRecords}` SET `comparisonInProgress`='0'");
		if ($db->ErrorNo()) {
			throw new \DBException("Cannot unmark table rows for comparison");
		}

		$db->Execute("UPDATE `{$DBTBspSettlementRecords}`
			SET `comparisonInProgress`='1'
			WHERE `status`='unmatched'
				AND `bspRecordId` IS NOT NULL
				AND `importedServiceId` IS NOT NULL
		");
		if ($db->ErrorNo()) {
			throw new \DBException("Cannot mark table rows for comparison");
		}



		$listObj = new \App\EntityList\BspSettlementRecords();
		$listObj->setColumns([
			'bspRecordId',
			'importedServiceId',
			'importedServiceIsRefund',
			'ticketNumber',
		]);
		$listObj->setFilter('comparisonInProgress', '=', '1');

		$this->updateStatus([
			'status' => sprintf('matchWithBspSettlements,initialize'),
		]);

		$query = $listObj->getFullSQL();
		$bspSettlementRecordsIterator = new \ModulesAbstract\PluginReports\Report\Model\Gatherer\DatabasePaginatedIterator($query['sql'], $query['bindVars']);

		$totalSettlementRecords = $listObj->getRecordsCount();

		$this->logger->info(sprintf("Found %s BSP settlement records unmatched. Compare them now", number_format($totalSettlementRecords)));

		$index = 0;
		$this->recordComparer = new \App\Bsp\Settlement\CompareWithImportedService();
		while ($bspSettlementRecord = $bspSettlementRecordsIterator->next()) {
			$index++;
			$this->updateStatus([
				'progress' => min(1.0, $index/$totalSettlementRecords),	// limit progress to 100% :) in cases where $index gets over the total. It might happen for some repetitive records
				'status' => sprintf('matchWithBspSettlements,process(IO service:%s, BSP service: %s)', $bspSettlementRecord['importedServiceId'], $bspSettlementRecord['bspRecordId']),
			]);

			$importedService = \Entity\ImportedService::select($bspSettlementRecord['importedServiceId']);
			$bspRecord = \Entity\BspRecord::select($bspSettlementRecord['bspRecordId']);

			/*
			 * TINA handles some relations differently than the GDS'es & BSP's does:
			 * 	TICK->REIS->RFND
			 * 		- in TINA are mapped to all the services involved (TICK, REIS, RFND)
			 * 		- in BSP the RFND is linked to the REIS only
			 *
			 * we should make sure we're handling the RFND for REIS identically on the BSP side
			 * */
			$extraData = [
				// this is a refund
				'isRefund' => 				(bool)$bspSettlementRecord['importedServiceIsRefund'],

				// we're comparing a RFND for a reissue with the RFND from the BSP
				'isRefundForReissue' => 	(bool)($importedService->idImportedServiceReissued && $bspSettlementRecord['importedServiceIsRefund']),

				// the ticket number used when doing comparisons
				'ticketNumber' => 			$bspRecord->ticketNumber,
			];

			list($statuses, $ticketStatus) = $this->compareRecords($bspRecord, $importedService, $extraData, $extraData);

			if ($statuses || $ticketStatus) {
				$row = [
					'key' => 		$bspSettlementRecord['key'],
				];

				if ($statuses) {
					$row['status'] = $statuses;
				}
				if ($ticketStatus) {
					$row['ticketStatus'] = $ticketStatus;
				}

				$this->queueBspSettlement($row);
			}
		}


		global $DBTBspSettlementRecords;
		$db = \DB::getInstance();
		$db->Execute("UPDATE `{$DBTBspSettlementRecords}` SET `comparisonInProgress`='0' WHERE `comparisonInProgress`='1'");
		if ($db->ErrorNo()) {
			throw new \DBException("Cannot mark table rows for comparison");
		}
	}

	/**
	 * Scan the importedOrder's services and pick the service with the specified ticket number
	 * @param \Entity\ImportedOrder $importedOrder
	 * @return \App\Entity\GeneralAbstract
	 */
	protected function pickImportedServiceFromOrder(\Entity\ImportedOrder $importedOrder, $ticketNumber)
	{
		$srv = reset(array_filter($importedOrder->object->services, function (\App\ImportedOrder\Service $srv) use ($ticketNumber) {
			return $srv->chosenOffer->confirmationNumber==$ticketNumber;
		}));

		/* @var $srv \App\ImportedOrder\Service */
		if (!$srv->idSystem) {
			throw new \Exception\Validator\EmptyValue("pickImportedServiceFromOrder failed");
		}
		$serviceDetails = \Entity\ImportedService::getDetailsByServiceExtSys($srv->idSystem, $importedOrder->idSystem, false);
		return \Entity\ImportedService::select($serviceDetails['id']);
	}

	/**
	 * Compare 2 records
	 * @return string[]
	 */
	protected function compareRecords($bspRecord, $importedService, $bspServiceExtraData, $importedServiceExtraData)
	{
		try {
			$comparison = $this->recordComparer->compare($bspRecord, $importedService, $bspServiceExtraData, $importedServiceExtraData);

			$rawComparisonStatuses = $comparison->getComparisonStatuses();

			// remap $rawComparisonStatuses from the class notation to the list of DB statuses
			$remapsList = [
				'ticketNumber:diff' => 				'diff:supplierData',
				'ticketDate:diff' => 				'diff:supplierData',
				'supplierCode3:diff' => 			'diff:supplier',

				'isIssue:diff' => 					'diff:operation',
				'isRefund:diff' => 					'diff:operation',
				'isVoid:diff' => 					'diff:operation',
				'isCc:diff' => 						'diff:price',

				'priceServiceCash:diff' => 			'diff:price',
				'priceServiceCc:diff' => 			'diff:price',
				'priceTaxCash:diff' => 				'diff:price',
				'priceTaxCc:diff' => 				'diff:price',
				'priceCommissionCash:diff' => 		'diff:price',
				'priceCommissionCc:diff' => 		'diff:price',
				'priceCommissionVATCash:diff' =>	'diff:price',
				'priceCommissionVATCc:diff' => 		'diff:price',
				'priceBalancePayable:diff' => 		'diff:price',
				'priceBalancePayable:identical' =>	'priceBalancePayable:identical',
			];

			$statuses = array_filter(array_unique(array_map(function($rawComparisonStatus) use ($remapsList) {
				return $remapsList[$rawComparisonStatus];
			}, $rawComparisonStatuses)));

			if (!$statuses) {
				$statuses[]= 'matched';
			}

			$ticketStatus = 'mixed';
			if ($comparison->isVoid->bspRow && $comparison->isVoid->importedService) {
				// a ticket has status 'voided' when is voided both in BSP & TINA
				$ticketStatus = 'voided';
			}
			elseif ($comparison->isRefund->bspRow && $comparison->isRefund->importedService) {
				$ticketStatus = 'refunded';
			}
			elseif ($comparison->isIssue->bspRow && $comparison->isIssue->importedService) {
				$ticketStatus = 'issued';
			}

			// printf(" %s %- 8s %s\n", $bspRecord->ticketNumber, $section, implode(",", $statuses));
			return [$statuses, $ticketStatus];
		}
		catch (\Exception\Validator\MismatchedValue $ex) {
			$this->logger->warning(sprintf("Cannot correctly handle prices for bsp record %s and importedService %s (ticket %s, probably the refund wasn't completly handled)", $bspRecord->getId(), $importedService->getId(), $bspRecord->ticketNumber));
			return [['diff:price', 'diff:operation', ], 'mixed'];
		}
	}


	/**
	 * Search the bspRecords list for a certain ticket number
	 */
	protected function searchBspRecords($ticketNumber, $airline, $minTicketDate, $isSale)
	{
		$listObj = new \App\EntityList\BspRecords();
		$listObj->setFilter('ticketNumber', '=', $ticketNumber);
		$listObj->setFilter('supplierCode3', '=', $airline);
		$listObj->setFilter('ticketDate', '>=', $minTicketDate);
		$listObj->setFilter('priceBalancePayableAmount', ($isSale?'>=':'<'), 0);
		$records = $listObj->exportRecords();
		return array_map(function($r) {
			return $r['id'];
		}, $records);
	}
}