Technical Note: [FortiSOAR / Cybersponse Tricks'n'Tips] Migrating records between CyOPs instances running different software versions

[Andy_G]

Description

 The following python script can be used to migrate records from one CyOPs instance to another, regardless of whether the two instances are on the same CyOPs version.

Prerequisites

1. Each module which is to be migrated must have the same fields, field names, and picklists defined in the destination CyOPs instance as are defined in the source CyOPs instance. To ensure this is the case, prior to running this script, those modules' configurations should be exported from the source CyOPs instance and imported into the destination CyOPs instance. This is done from the respective CyOPs instances' Configuration Managers.
2. The constants at the start of the script (lines 6-9) must be defined. These constants will be the username and password to use on both CyOPs instances, and the respective hostnames for each.
3. The list of modules to be migrated (lines13-16) must be populated properly. The list can be modified to include whatever modules you wish to move between CyOPs instances, and as an example in the script it is pre-populated to migrate the Users, Alerts, and the Incidents modules. NOTE: When populating this list, make note of modules that have "Lookup" fields. If Module A has a Lookup field which looks up a record in Module B, then records from Module B MUST be migrated prior to Module A's migration. This can be done by placing Module B before Module A in the modules list. One common example of this case is the "Assigned To" field on the Alerts module, which is a lookup for the Users module. Because the Alerts which will be migrated will be Assigned to Users which may not exist on the destination CyOPs instance, the Users records must be migrated first

import requests
import json
import math

# VALUES NEEDED:
USERNAME = ""
PASSWORD = ""
SRC_HOSTNAME = ""
DEST_HOSTNAME = ""
LIMIT = 100
CHUNK_SIZE = 5

modules = [
    'people',
    'alerts',
    'incidents'
]

class Migrator:
    def __init__(self, username, password):
        self.source_hostname = SRC_HOSTNAME
        self.destination_hostname = DEST_HOSTNAME
        self.credentials = {
                            'credentials': {
                                'loginid': username,
                                'password': password
                            }
        }
        self.src_token = None
        self.dst_token = None
        self._authorize()
        self.records=[]
        self.skipped=[]
        self.limit = LIMIT
        self.chunksize = CHUNK_SIZE


    def _authorize(self):
        src_auth_response = requests.post('https://{}/auth/authenticate'.format(self.source_hostname),
                                      data=json.dumps(self.credentials),
                                      headers={},
                                      verify=False)
        self.src_token = json.loads(src_auth_response.text)['token']
        self.src_headers = {"Authorization": "Bearer {}".format(self.src_token)}

        dst_auth_response = requests.post('https://{}/auth/authenticate'.format(self.destination_hostname),
                                          data=json.dumps(self.credentials),
                                          headers={},
                                          verify=False)
        print("Auth: old token is {}".format(self.dst_token))
        self.dst_token = json.loads(dst_auth_response.text)['token']
        self.dst_headers = {"Authorization": "Bearer {}".format(self.dst_token)}
        print("Auth: new token is {}".format(self.dst_token))
        return True


    def migrate_all_records(self, module, page):
        print("Exporting {}, page {}".format(module, page))
        record_request = requests.get('https://{}/api/3/{}?$limit={}&$page={}&$export=true'.format(self.source_hostname,
                                                                                          module,
                                                                                          self.limit,
                                                                                          page),
                                      headers=self.src_headers,
                                      verify=False)
        if record_request.status_code == 401:
            self._authorize()
            record_request = requests.get('https://{}/api/3/{}?$limit={}&$page={}&$export=true'.format(self.source_hostname,
                                                                                              module,
                                                                                              self.limit,
                                                                                              page),
                                          headers=self.src_headers,
                                          verify=False)
        if record_request.status_code == 404:
            print("{} not found. Skipping...".format(module))
            self.skipped.append(module)
            return False
        record_request=record_request.json()
        pages = math.ceil(record_request['hydra:totalItems'] / self.limit)
        self.records.extend(record_request['hydra:member'])
        print("Exported {} total records...".format(len(self.records)))
        self.import_all_records()
        
        if pages > page:
            self.migrate_all_records(module, page+1)
        
        return True

    def import_all_records(self):
        pages = math.ceil(len(self.records) / self.chunksize)
        for page in range(1,pages+1):
            if page == 1:
                records = self.records[:page * self.chunksize]
            elif page == pages:
                records = self.records[(page-1)*self.chunksize:]
            else:
                records = self.records[(page-1)*self.chunksize:page*self.chunksize]

            import_request = requests.post('https://{}/api/3/'.format(self.destination_hostname),
                                           json=records,
                                           headers=self.dst_headers,
                                           verify=False)
            if import_request.status_code == 401:
                self._authorize()
                import_request = requests.post('https://{}/api/3/'.format(self.destination_hostname),
                                               json=records,
                                               headers=self.dst_headers,
                                               verify=False)
            print("Import Response: {}".format(import_request.status_code))
            #print(import_request.text)
            print("Imported {} records...".format(page*self.chunksize))
        self.records = []
        return True


    def check_quantity(self, module):
        get_module = requests.get('https://{}/api/3/{}'.format(self.destination_hostname, module),
                                  headers=self.dst_headers,
                                  verify=False)
        if get_module.status_code == 401:
            self._authorize()
            get_module = requests.get('https://{}/api/3/{}'.format(self.destination_hostname, module),
                                      headers=self.dst_headers,
                                      verify=False)
        if get_module.status_code == 404:
            print("{} not found. Skipping...".format(module))
            self.skipped.append(module)
            return False
        quantity = get_module.json()['hydra:totalItems']
        return quantity

mig = Migrator(USERNAME, PASSWORD)
for module in modules:
    completed = mig.migrate_all_records(module, 1)
for module in mig.skipped:
    print("Module {} was not found and was skipped".format(module))