<?php

namespace App\Cron\Job;


class GdsSerializationUpgrade extends \App\Cron\Job\Utils\LongRunningCronJobAbstract
{
	/**
	 * Cronjob category: {system|operational|financial|reporting}
	 * @var string
	 */
	protected $category = 'system';

	/**
     * {@inheritDoc}
     * @see \App\Cron\Job\Utils\GeneralAbstract::validateSettings()
     */
    public function validateSettings(\App\Entity\Messages $msgs, array $fields)
    {
        if ($fields['scheduleOptions']['type'] != 'scheduled') {
            $msgs->addWarningText("The process should be set as `once per day or less`.", 'type');
        }
    }

	/**
	 * (non-PHPdoc)
	 * @see \App\Cron\Job\Utils\GeneralAbstract::executeJob()
	 */
	protected function executeJob()
	{
		// disable entity cache, it might end up using a log of RAM
		\App\Entity\Cache::getInstance()->flush();
		\App\Entity\Cache::getInstance()->flushInternal();
		\App\Entity\Cache::getInstance()->setCacheObject( new \Engine\Cache\Dummy() );

		// disable user trails. It might lead to high RAM usage
		\App\UserTrail::getInstance()->storeTrail(false);

		$scannedSystems = array_filter([
			// core GDS
			((bool)\Entity\ExtResSystem::getSystemIdByCode('amadeus')?'amadeus':''),
			((bool)\Entity\ExtResSystem::getSystemIdByCode('sabre')?'sabre':''),
			((bool)\Entity\ExtResSystem::getSystemIdByCode('worldspan')?'worldspan':''),
			((bool)\Entity\ExtResSystem::getSystemIdByCode('galileo')?'galileo':''),
			((bool)\Entity\ExtResSystem::getSystemIdByCode('apollo')?'apollo':''),

			// // plugins
			((bool)\Entity\ExtResSystem::getSystemIdByCode('fareLogixSystem')?'fareLogixSystem':''),
			((bool)\Entity\ExtResSystem::getSystemIdByCode('airGateway')?'airGateway':''),
			((bool)\Entity\ExtResSystem::getSystemIdByCode('nsrail')?'nsrail':''),
		]);

		$systemsProgress = array_combine($scannedSystems, array_fill(0, count($scannedSystems), 0));


		$this->updateStatus([
			'progressSteps' => 	count($scannedSystems),
			'systemsProgress' => $systemsProgress,
		]);

		if (in_array('amadeus', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	0,	'progress' => 		0, 'currentStep' => 'convertAmadeus', 'systemsProgress' => $systemsProgress, ]);
			$this->convertAmadeus();
			$systemsProgress['amadeus'] = 1;
		}

		if (in_array('sabre', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	1,	'progress' => 		0, 'currentStep' => 'convertSabre', 'systemsProgress' => $systemsProgress, ]);
			$this->convertSabre();
			$systemsProgress['sabre'] = 1;
		}

		if (in_array('worldspan', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	2,	'progress' => 		0, 'currentStep' => 'convertWorldspan', 'systemsProgress' => $systemsProgress, ]);
			$this->convertWorldspan();
			$systemsProgress['worldspan'] = 1;
		}

		if (in_array('galileo', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	3,	'progress' => 		0, 'currentStep' => 'convertGalileo', 'systemsProgress' => $systemsProgress, ]);
			$this->convertGalileo();
			$systemsProgress['galileo'] = 1;
		}

		if (in_array('apollo', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	4,	'progress' => 		0, 'currentStep' => 'convertApollo', 'systemsProgress' => $systemsProgress, ]);
			$this->convertApollo();
			$systemsProgress['apollo'] = 1;
		}

		if (in_array('fareLogixSystem', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	5,	'progress' => 		0, 'currentStep' => 'convertFarelogix', 'systemsProgress' => $systemsProgress, ]);
			$this->convertFarelogix();
			$systemsProgress['fareLogixSystem'] = 1;
		}

		if (in_array('airGateway', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	6,	'progress' => 		0, 'currentStep' => 'convertAirGateway', 'systemsProgress' => $systemsProgress, ]);
			$this->convertAirGateway();
			$systemsProgress['airGateway'] = 1;
		}

		if (in_array('nsrail', $scannedSystems)) {
			$this->updateStatus(['progressStep' => 	7,	'progress' => 		0, 'currentStep' => 'convertNsrail', 'systemsProgress' => $systemsProgress, ]);
			$this->convertNsrail();
			$systemsProgress['nsrail'] = 1;
		}
		$this->updateStatus(['systemsProgress' => $systemsProgress, ]);

	}


    /**
     * Build a paginated iterator for records, common code for all GDS convertors
     *
     * @param string $tableName
     * @param string $likeField
     * @param string $idField
     * @return array{0:?\ModulesAbstract\PluginReports\Report\Model\Gatherer\DatabasePaginatedIterator,1:int,2:int}
     */
    protected function buildRecordsIterator(string $tableName, string $likeField, string $idField = 'id'): array
    {
		// // v1: optimized SELECT, but it places a lot of burden on the DB. This will probably lock BIG DB's
        // // total count of records to be upgraded
        // $queryTotal = "SELECT COUNT(*) FROM `{$tableName}` WHERE (`{$likeField}` LIKE '{\"serializer\"%' )";
        // $total = \DB::getInstance()->GetOne($queryTotal);

        // $db = \DB::getInstance();
        // // first id to start from; avoid using dynamic data in WHERE besides id
        // $queryMinId = "SELECT MIN(`{$idField}`) FROM `{$tableName}` WHERE (`{$likeField}` LIKE '{\"serializer\"%' )";
        // $minUnconvertedId = $db->GetOne($queryMinId);
        // if ($db->ErrorNo()) {
        //     throw new \DBException("Cannot get min id for table {$tableName}");
        // }



		// v2: non-optimized SELECT, but it places less burden on the DB. This should slow down conversion resuming considerably
        // // total count of records to be upgraded
        // $queryTotal = "SELECT COUNT(*) FROM `{$tableName}` WHERE 1";
        // $total = \DB::getInstance()->GetOne($queryTotal);

        // $db = \DB::getInstance();
        // // first id to start from; avoid using dynamic data in WHERE besides id
        // $queryMinId = "SELECT MIN(`{$idField}`) FROM `{$tableName}` WHERE 1";
        // $minUnconvertedId = $db->GetOne($queryMinId);
        // if ($db->ErrorNo()) {
        //     throw new \DBException("Cannot get min id for table {$tableName}");
        // }

		// v3: non-optimized select, but it places less burden on the DB. This should slow down conversion resuming considerably. This only "approximates" the number of rows to process
		// total count of records to be upgraded
        $queryTotal = "SELECT MAX(`{$idField}`)-MIN(`{$idField}`)+1 FROM `{$tableName}` WHERE 1";
        $total = \DB::getInstance()->GetOne($queryTotal);

        $db = \DB::getInstance();
        // first id to start from; avoid using dynamic data in WHERE besides id
        $queryMinId = "SELECT MIN(`{$idField}`) FROM `{$tableName}` WHERE 1";
        $minUnconvertedId = $db->GetOne($queryMinId);
        if ($db->ErrorNo()) {
            throw new \DBException("Cannot get min id for table {$tableName}");
        }


		// COMMON PATHS
		// if we have no data, do a fast return
        if (!$minUnconvertedId) {
            return [null, 0, 0];
        }

        // stable, reproducible pagination query
        $query = "SELECT `{$idField}` AS `id`, `{$likeField}` AS `dataField` FROM `{$tableName}` WHERE `{$idField}`>={$minUnconvertedId} ORDER BY `{$idField}` ASC";
        $queryParams = [];
        $iterator = new \ModulesAbstract\PluginReports\Report\Model\Gatherer\DatabasePaginatedIterator($query, $queryParams);
        $iterator->setOption('queueLength', 10000);

        return [$iterator, $minUnconvertedId, $total, ];
	}

	/**
	 * Iterate through a paginated iterator, update status, common code for all GDS convertors
	 *
	 * @param \ModulesAbstract\PluginReports\Report\Model\Gatherer\DatabasePaginatedIterator $iterator
	 * @param int $total
	 * @param int $minUnconvertedId
	 * @param callable($id):void $convertCallback
	 */
	protected function processRecordsIterator($iterator, int $total, int $minUnconvertedId, callable $convertCallback): void
	{
		$status = [
			'scan' => [
				'total' => $total,
				'index' => 0,
				'ignored' => 0,
				'converted' => 0,
			]
		];
		// Skip rows that do not start with the expected serializer marker
		$startPattern = '{"serializer":"Engine\\\\Serializer\\\\Custom",';

		$this->logger->info(sprintf("Found approximately %d records to upgrade serialization, starting to scan from id %s", $total, $minUnconvertedId));
		$startedConversion = false;
		foreach ($iterator->asIterator() as $row) {
			$status['scan']['index']++;
			$this->updateStatus([
				'progress' => $status['scan']['index']/$status['scan']['total'],
			]);

			if (substr($row['dataField'], 0, strlen($startPattern)) !== $startPattern) {
				$status['scan']['ignored']++;
				continue;
			}
			if (!$startedConversion) {
				$this->logger->info(sprintf("Starting conversion from id %s", $row['id']));
			}

			$startedConversion = true;

			\DB::startTransaction();
			$convertCallback($row['id']);
			\DB::commitTransaction();

			$status['scan']['converted']++;
		}

		$this->logger->success(sprintf("Finished serialization upgrade"));
		$this->logger->success(sprintf("Scanned %s records, ignored %s records, converted %s records. Found %s index gaps",
			$status['scan']['index'],
			$status['scan']['ignored'],
			$status['scan']['converted'],
			$status['scan']['total'] - ($status['scan']['ignored'] + $status['scan']['converted'])
		));
	}

	protected function convertAmadeus()
	{
		$this->logger->info("> Starting serialization upgrade for Amadeus");

		global $DBTAmadeusServicesData;
		[$amadeusRecordsIterator, $minUnconvertedId, $amadeusRecordsTotal] = $this->buildRecordsIterator($DBTAmadeusServicesData, 'transactionDocData');
		if (!$amadeusRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
            $amadeusRecordsIterator,
            $amadeusRecordsTotal,
            $minUnconvertedId,
            function ($id) { $this->convertAmadeusService($id); }
        );

    }

	protected function convertAmadeusService($amadeusRecordId)
	{
		$service = \Entity\AmadeusService::select($amadeusRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionParseLog',
					'transactionProcessLog',
					'transactionDocData',
					'transactionSupplierData',
					'transactionPriceData',
					'transactionPassengerData',
					'transactionSegmentData',
					'transactionExtraData',
					'transactionRemarksData',
					'transactionBookingAgentData',
					'transactionIssueAgentData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);


				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $amadeusRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$amadeusRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTAmadeusServicesData;
		$updateSQL = "UPDATE `$DBTAmadeusServicesData` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $amadeusRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'AmadeusService' in the database, primaryKeyValue: '%s'", $amadeusRecordId), $updateSQL, $saveData);
		}
	}


	protected function convertSabre()
	{
		$this->logger->info("> Starting serialization upgrade for Sabre");

		global $DBTSabreServices;
		[$sabreRecordsIterator, $minUnconvertedId, $sabreRecordsTotal] = $this->buildRecordsIterator($DBTSabreServices, 'transactionData');
		if (!$sabreRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
			$sabreRecordsIterator,
			$sabreRecordsTotal,
			$minUnconvertedId,
			function ($id) { $this->convertSabreService($id); }
		);

	}


	protected function convertSabreService($sabreRecordId)
	{
		$service = \Entity\SabreService::select($sabreRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);

				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $sabreRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$sabreRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// if I log these, I'll get exceptions when trying to serialize the messagesCollector objects, so I disabled it
						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTSabreServices;
		$updateSQL = "UPDATE `$DBTSabreServices` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $sabreRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'SabreService' in the database, primaryKeyValue: '%s'", $sabreRecordId), $updateSQL, $saveData);
		}
	}


	protected function convertWorldspan()
	{
		$this->logger->info("> Starting serialization upgrade for Worldspan");

		global $DBTWorldspanServices;
		[$worldspanRecordsIterator, $minUnconvertedId, $worldspanRecordsTotal] = $this->buildRecordsIterator($DBTWorldspanServices, 'transactionData');
		if (!$worldspanRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
			$worldspanRecordsIterator,
			$worldspanRecordsTotal,
			$minUnconvertedId,
			function ($id) { $this->convertWorldspanService($id); }
		);

	}


	protected function convertWorldspanService($worldspanRecordId)
	{
		$service = \Entity\WorldspanService::select($worldspanRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);

				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $worldspanRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$worldspanRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// if I log these, I'll get exceptions when trying to serialize the messagesCollector objects, so I disabled it
						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTWorldspanServices;
		$updateSQL = "UPDATE `$DBTWorldspanServices` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $worldspanRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'WorldspanService' in the database, primaryKeyValue: '%s'", $worldspanRecordId), $updateSQL, $saveData);
		}
	}


	protected function convertGalileo()
	{
		$this->logger->info("> Starting serialization upgrade for Galileo");

		global $DBTGalileoServices;
		[$galileoRecordsIterator, $minUnconvertedId, $galileoRecordsTotal] = $this->buildRecordsIterator($DBTGalileoServices, 'transactionData');
		if (!$galileoRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
			$galileoRecordsIterator,
			$galileoRecordsTotal,
			$minUnconvertedId,
			function ($id) { $this->convertGalileoService($id); }
		);

	}


	protected function convertGalileoService($galileoRecordId)
	{
		$service = \Entity\GalileoService::select($galileoRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);

				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $galileoRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$galileoRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// if I log these, I'll get exceptions when trying to serialize the messagesCollector objects, so I disabled it
						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTGalileoServices;
		$updateSQL = "UPDATE `$DBTGalileoServices` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $galileoRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'GalileoService' in the database, primaryKeyValue: '%s'", $galileoRecordId), $updateSQL, $saveData);
		}
	}


	protected function convertApollo()
	{
		$this->logger->info("> Starting serialization upgrade for Apollo");

		global $DBTApolloServices;
		[$apolloRecordsIterator, $minUnconvertedId, $apolloRecordsTotal] = $this->buildRecordsIterator($DBTApolloServices, 'transactionData');
		if (!$apolloRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
			$apolloRecordsIterator,
			$apolloRecordsTotal,
			$minUnconvertedId,
			function ($id) { $this->convertApolloService($id); }
		);

	}


	protected function convertApolloService($apolloRecordId)
	{
		$service = \Entity\ApolloService::select($apolloRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);

				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $apolloRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$apolloRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// if I log these, I'll get exceptions when trying to serialize the messagesCollector objects, so I disabled it
						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTApolloServices;
		$updateSQL = "UPDATE `$DBTApolloServices` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $apolloRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'ApolloService' in the database, primaryKeyValue: '%s'", $apolloRecordId), $updateSQL, $saveData);
		}
	}


	protected function convertFarelogix()
	{
		$this->logger->info("> Starting serialization upgrade for Farelogix");

		global $DBTPluginFarelogixImportServices;
		[$farelogixRecordsIterator, $minUnconvertedId, $farelogixRecordsTotal] = $this->buildRecordsIterator($DBTPluginFarelogixImportServices, 'transactionData');
		if (!$farelogixRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
			$farelogixRecordsIterator,
			$farelogixRecordsTotal,
			$minUnconvertedId,
			function ($id) { $this->convertFarelogixService($id); }
		);

	}


	protected function convertFarelogixService($farelogixRecordId)
	{
		$service = \Entity\FarelogixService::select($farelogixRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);

				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $farelogixRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$farelogixRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// if I log these, I'll get exceptions when trying to serialize the messagesCollector objects, so I disabled it
						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTPluginFarelogixImportServices;
		$updateSQL = "UPDATE `$DBTPluginFarelogixImportServices` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $farelogixRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'FarelogixService' in the database, primaryKeyValue: '%s'", $farelogixRecordId), $updateSQL, $saveData);
		}
	}


	protected function convertAirGateway()
	{
		$this->logger->info("> Starting serialization upgrade for AirGateway");

		global $DBTPluginAirGatewayImportServices;
		[$airgatewayRecordsIterator, $minUnconvertedId, $airgatewayRecordsTotal] = $this->buildRecordsIterator($DBTPluginAirGatewayImportServices, 'transactionData');
		if (!$airgatewayRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
			$airgatewayRecordsIterator,
			$airgatewayRecordsTotal,
			$minUnconvertedId,
			function ($id) { $this->convertAirGatewayService($id); }
		);

	}


	protected function convertAirGatewayService($airgatewayRecordId)
	{
		$service = \Entity\AirGatewayService::select($airgatewayRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);

				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $airgatewayRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$airgatewayRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// if I log these, I'll get exceptions when trying to serialize the messagesCollector objects, so I disabled it
						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTPluginAirGatewayImportServices;
		$updateSQL = "UPDATE `$DBTPluginAirGatewayImportServices` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $airgatewayRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'AirGatewayService' in the database, primaryKeyValue: '%s'", $airgatewayRecordId), $updateSQL, $saveData);
		}
	}


	protected function convertNsrail()
	{
		$this->logger->info("> Starting serialization upgrade for NsRail");

		global $DBTPluginNsRailImportServices;
		[$nsrailRecordsIterator, $minUnconvertedId, $nsrailRecordsTotal] = $this->buildRecordsIterator($DBTPluginNsRailImportServices, 'transactionData');
		if (!$nsrailRecordsTotal) {
			$this->logger->idle("Nothing to convert");
			return;
		}

		$this->processRecordsIterator(
			$nsrailRecordsIterator,
			$nsrailRecordsTotal,
			$minUnconvertedId,
			function ($id) { $this->convertNsrailService($id); }
		);

	}


	protected function convertNsrailService($nsrailRecordId)
	{
		$service = \Entity\NsRailService::select($nsrailRecordId);		// load from DB
		$serializer = new \Engine\Serializer\Json();
		$saveData = [];
		foreach([	'transactionReferences',
					'transactionData',
					'extraData'] as $field) {

			if ($service->{$field}) {
				$originalData = $service->{$field};
				$saveData[$field] = $serializer->serialize($originalData);

				$diff = [];
				if (!compareMixedTypes($originalData, $serializer->unserialize($saveData[$field]), $diff, '')) {
					$this->logger->critical(sprintf("Object field %s is different for id %s! %s on %s", $field, $nsrailRecordId, $diff['reason'], ($diff['path']?:'root')));
					\Log::getInstance()->critical(sprintf("[%s] Object comparison failed! %s on %s", __METHOD__, $diff['reason'], ($diff['path']?:'root')), [
						'function' => 		__FUNCTION__,
						'field' => 			$field,
						'id' => 			$nsrailRecordId,
						'diff' => 			new \Engine\Logger\Context\ArrayAssoc($diff),

						// if I log these, I'll get exceptions when trying to serialize the messagesCollector objects, so I disabled it
						// 'object' => 		new \Engine\Logger\Context\Auto($originalData),
						// 'serialized' => 	new \Engine\Logger\Context\Auto($serializer->unserialize($saveData[$field])),
					]);
					throw new \RuntimeException(sprintf("Object comparison failed!"));
				}
			}
		}

		// direct write in the DB
		$db = \DB::getInstance();
		global $DBTPluginAirGatewayImportServices;
		$updateSQL = "UPDATE `$DBTPluginAirGatewayImportServices` SET " .
			// list all columns in here
			implode(", ", array_map(function($c) {
				return "`$c`=?";
			}, array_keys($saveData))) .
		" WHERE `id`=?";

		$saveData['id'] = $nsrailRecordId;
		$db->Execute($updateSQL, $saveData);
		if ($db->ErrorNo()) {	// execution failed
			throw new \DBException(sprintf("Could not update entity 'NsrailService' in the database, primaryKeyValue: '%s'", $nsrailRecordId), $updateSQL, $saveData);
		}
	}

}
