# -*- coding: utf-8 -*
import os
import json
import ftplib
import paramiko
import importlib.resources
import glaciation.devices as rs
from stat import S_ISDIR, S_ISREG
from abc import ABCMeta, abstractmethod
from glaciation.helper import measureTime
from multiprocessing import Process, Queue
import glaciation.devices.uploader as uploader
from glaciation.networking import checkPortIsOpen
JSON_DEFAULT_CONN = importlib.resources.open_text(rs, "defaultConnections.json").name
[docs]@measureTime
def loadConfs(ips, devType, confFolder, multiProcess=True, defaultConns=None, queue=False):
"""Search available method to upload and
upload files from confFolder recursively to devices in ips addresses
posibility to delete files in the remote device inserting a file named __delete__ with file paths to be deleted
Parameters
----------
ips : list/string
list ip address of the target devices.
If a list is given, each ip address is going to be executed in a new Process in parallel (Multiprocessing)
If ips is not a list, the function is not going to start a new Process (Thread)
devType : fixed string (Vega, HMI_S, TSW)
In case of list it will call itself with each type
If a list is given confFolder must be a list as well
Example without list: loadConfs([ips1], "Vega", CONF, multiProcess=True)
Example with list: loadConfs([ips1,ips2], ["Vega","TSW"], [CONF, CONF_TSW], multiProcess=True)
confFolder : string
Folder containing the files to upload
multiProcess : bool
If True all configurations will run in a new process in parallel (default is True)
defaultConns : string
Path to default connection json. The default connections json file will be override
queue : string
For internal use only, to share memory between processes (default is False)
Raises
------
Exception
loadConfError
Examples
| CONF = /path/to/confFolder
| CONF_TSW = /path/to/confFolder2
| ips1 = ["172.16.5.1","172.16.5.2","172.16.5.1","172.16.5.2"]
| ips2 = ["172.16.3.1","172.16.3.2","172.16.4.1","172.16.4.2"]
| loadConfs(ips1, "Vega", CONF, multiProcess=False)
| loadConfs([ips1,ips2], ["Vega","TSW"], [CONF, CONF_TSW] , multiProcess=True)
"""
processes = []
queues = []
# if only one ip convert it to list
if type(ips) != list:
ips = [ips]
# option to override default connections json
if defaultConns is not None:
JSON_DEFAULT_CONN = defaultConns
# foreach ip in the list start a new Process to upload in parallel
x = 0
for ip in ips:
if multiProcess:
q = Queue()
# if multiple device types are received call this function again with each type
if type(devType) == type(confFolder) == list:
# Recursive call
p = Process(target=loadConfs, args=[ips[x], devType[x], confFolder[x], multiProcess, q])
else:
# Call in a new ProcessR
p = Process(target=__loadOneConf, args=[ip, devType, confFolder, q])
p.start()
processes.append(p)
queues.append(q)
else:
if type(ips) == type(devType) == type(confFolder) == list:
# Recursive call
loadConfs(ips[x], devType[x], confFolder[x], multiProcess)
else:
__loadOneConf(ip, devType, confFolder)
x += 1
print("Waiting for all uploads to complete")
for process in processes:
process.join()
# print("Check if any upload process has raise errors")
errors = ""
for q in queues:
message = q.get()
if message is not None:
errors += message + "\n"
if errors == "":
errors = None
if queue is not False: # recursive call response
queue.put(errors)
return
if errors is not None:
raise Exception(errors)
# recursive call response
print("----- Successfully Uploaded all confs -----")
def __loadOneConf(ip, devType, confFolder, queue=False):
"""
:meta private:
Search available method to upload and
upload files from confFolder recursively to device in ip address
@param ip: ip address of the target device.
@param devType: Device type (Vega, HMI_S, TSW)
@param confFolder: Folder containing the files to upload
@param queue: For internal use only, to share memory between processes
@return: None
@raise loadConfError: raises an exception
"""
try:
print("Loading ip %s of type %s" % (ip, devType))
# get correct selector to upload
uploader = __uploaderSelector(ip, devType, JSON_DEFAULT_CONN)
# upload files
__loadConfToDevice(uploader, confFolder)
# close all connections
uploader.exit()
# Tell parent process everything went ok
if queue is not False:
queue.put(None)
except Exception as e:
print("LoadConfs error: %s" % str(e))
if queue is not False:
queue.put("Failed loading ip %s of type %s: %s" % (ip, devType, str(e)))
else:
raise Exception("Failed loading ip %s of type %s: %s" % (ip, devType, str(e)))
def __loadConfToDevice(uploader, confFolder):
"""
:meta private:
Load configuration in a specific device
@param uploader: Uploader class Instance (Vega, HMI_S, TSW)
@param confFolder: Folder containing the files to upload
If confFolder contains a file name __delete__,
this function will not upload that file to server
and it will delete files written in that file.
Example of __delete__ file content:
./home
./platform.cfg
@return: None
@raise loadConfToDevice: raises an exception
"""
if not os.path.exists(confFolder):
raise Exception("loadConf: Configuration folder does not exist")
for dirPath, dirNames, fileNames in os.walk(confFolder):
# get relative path
relativePath = os.path.relpath(dirPath, confFolder)
# create directories
for dir in dirNames:
destFolderPath = os.path.join(relativePath, dir).replace(os.sep, "/")
destPath = os.path.split(destFolderPath)[0]
# create folder in the target device
uploader.createFolder(dir, destPath, destFolderPath)
# upload files
for fileName in fileNames:
# get file to upload
filepath = os.path.join(dirPath, fileName).replace(os.sep, "/")
# delete all files in __delete__ file
if "__delete__" in fileName:
with open(filepath) as file:
filesToDel = [line.rstrip() for line in file]
uploader.deleteFiles(filesToDel)
else:
# try to upload file to device
uploader.uploadFile(fileName, filepath, relativePath)
def __uploaderSelector(ip, devType, connJson):
'''
:meta private:
Search for the correct connection for available options for each device type
@param devType: Device type (Vega, HMI_S, TSW)
@param connJson: Json file containing candidate connections
@return Uploader class instance
'''
connInstance = False
jsonFilePath = os.path.dirname(connJson)
# open json file with connection list
with open(connJson) as f:
candidateConns = json.load(f)
# for each connection of type devType try to connect
for conn in candidateConns[devType]:
closedPorts = []
print("Checking connection on port %s, protocol %s, user %s" % (conn["port"], conn["protocol"], conn["user"]))
# If port closed go to next conn
if (conn["port"] not in closedPorts) and (not checkPortIsOpen(ip, conn["port"])):
closedPorts.append(conn["port"])
continue
# If ftp
if conn["protocol"] == "ftp":
try:
connInstance = Ftp(ip, conn["user"].encode("utf-8"), conn["pwd"].encode("utf-8"), conn["port"])
break
except Exception as e:
print("Connection NOT OK")
connInstance = False
# If sftp
elif conn["protocol"] == "sftp":
disAlgorithms = conn["disAlgorithms"] if "disAlgorithms" in conn else None
timeout = conn["timeout"] if "timeout" in conn else 3 # default timeout 3s
try:
keyFullPath = jsonFilePath + '/' + conn["key"]
connInstance = SFtp(ip, conn["user"], keyFullPath, conn["pwd"], conn["port"], disAlgorithms, timeout)
break
except Exception as e:
print("Connection NOT OK: %s" % str(e))
connInstance = False
# If connections not ok
if connInstance == False:
raise Exception("None of the connections available was valid for %s" % ip)
# select correct subclass of class Uploader (VEGA, HMI_S, TSW)
selectedClass = globals()[devType]
print("Connection of %s is OK. Uploader has been selected." % ip)
uploaderClass = selectedClass(connInstance)
# upload all files to this path (for HMI_S)
if "upload_path" in conn:
uploaderClass.setUploadPath(conn["upload_path"])
return uploaderClass
class Uploader():
"""
:meta private:
"""
__metaclass__ = ABCMeta
@abstractmethod
def __init__(self, connection):
'''
@param connection: Connection class instance (Ftp, SFtp)
'''
self.conn = connection
self.uploadPath = ''
def setUploadPath(self, path):
self.uploadPath = path
@abstractmethod
def createFolder(self, folderName, devPath, destFolder):
'''
@param folderName: Folder name to create
@param devPath: device path to check if folder exists
@param destFolder: Folder relative path from root
'''
existingFolders = self.conn.showFolders(devPath)
# if dir does not exist, create it
if (folderName not in existingFolders):
self.conn.createFolder(destFolder)
@abstractmethod
def uploadFile(self, fileName, origFilePath, destPath):
'''
@param fileName: Name of the file
@param origFilePath: File path to be uploaded
@param destPath: File relative full path from root
'''
destFilePath = os.path.join(destPath, fileName).replace(os.sep, "/")
self.conn.uploadFile(origFilePath, destFilePath)
@abstractmethod
def deleteFiles(self, files):
self.conn.deleteFiles(files)
@abstractmethod
def exit(self):
self.conn.exit()
class Vega(Uploader):
"""
:meta private:
Subclass of class Uploader
"""
def __init__(self, connection):
super(Vega, self).__init__(connection)
def createFolder(self, folderName, devPath, destFolder):
'''
@param folderName: Folder name to create
@param devPath: device path to check if folder exists
@param destFolder: Folder relative path from root
'''
print("Creating folder %s" % destFolder)
existingFolders = self.conn.showFolders(devPath)
# if dir does not exist, create it
# In Vega SLOT1 Folder not showing sometimes
if (folderName not in existingFolders) and ("SLOT" not in destFolder):
self.conn.createFolder(destFolder)
def uploadFile(self, fileName, origFilePath, destPath):
'''
@param fileName: Name of the file
@param origFilePath: File path to be uploaded
@param destPath: File relative full path from root
'''
print("Trying to upload file %s" % fileName)
# rename the aplication img to app.bin
if fileName == "CCU_app.int.img" or fileName == "dev_CCU_app.int.img":
destFilePath = os.path.join(destPath, "app.bin").replace(os.sep, "/")
else:
destFilePath = os.path.join(destPath, fileName).replace(os.sep, "/")
# avoid log upload, because the logs can not be overwritten or erased
if ".LOG" in fileName:
print("Log files not permmited %s" % fileName)
return
try:
self.conn.uploadFile(origFilePath, destFilePath)
except Exception as e:
# cuando no deja cargar por ftp el borrar los logs suele resolver el error
print("Upload to Vega failed: %s" % str(e))
'''
print("Erasing all logs and trying again..")
self.conn.deleteFiles([
"FWK_REPORT.LOG", "SYSTEM_REPORT.LOG", "NVRAM/FWK_REPORT.LOG",
"NVRAM/FWK_CDU_REPORT.LOG"
])
self.conn.uploadFile(origFilePath, destPath)
'''
def deleteFiles(self, files):
super(Vega, self).deleteFiles(files)
def exit(self):
super(Vega, self).exit()
class TSW(Uploader):
"""
:meta private:
TSW subclass of class Uploader
"""
def __init__(self, connection):
super(TSW, self).__init__(connection)
def createFolder(self, folderName, devPath, destFolder):
super(TSW, self).createFolder(folderName, devPath, destFolder)
def uploadFile(self, fileName, origFilePath, destPath):
super(TSW, self).uploadFile(fileName, origFilePath, destPath)
def deleteFiles(self, files):
super(TSW, self).deleteFiles(files)
def exit(self):
super(TSW, self).exit()
class HMI_S(Uploader):
"""
:meta private:
HMI_S subclass of class Uploader
HMI Safe (Artech)
"""
def __init__(self, connection):
super(HMI_S, self).__init__(connection)
def createFolder(self, folderName, devPath, destFolder):
super(HMI_S, self).createFolder(folderName, devPath, destFolder)
def uploadFile(self, fileName, origFilePath, destPath):
'''
Usuario: maintainer
Fichero de claves: TCMS_SAFE\CONFS\Delta1\Conf23\HMIS\ClavesUsuarios\maintainer.*
Ruta de carga: /home/maintainer/uploads
Usuario: developer
Fichero de claves: TCMS_SAFE\CONFS\Delta1\Conf23\HMIS\ClavesUsuarios\developer.*
Ruta de carga: /home/developer
Cuando no hay nada configurado y se carga por primera vez el tar.gz:
Usuario: developer
Password: eicab
Ruta de carga: /uploads/eiCAB_sw_update
'''
destPath = os.path.join(self.uploadPath, destPath)
super(HMI_S, self).uploadFile(fileName, origFilePath, destPath)
def deleteFiles(self, files):
super(HMI_S, self).deleteFiles(files)
def exit(self):
super(HMI_S, self).exit()
class Connection():
"""
:meta private:
"""
__metaclass__ = ABCMeta
@abstractmethod
def showFolders(self, path):
pass
@abstractmethod
def createFolder(self, path):
pass
@abstractmethod
def uploadFile(self, originFilePath, destPath):
pass
@abstractmethod
def deleteFiles(self, files):
pass
@abstractmethod
def exit(self):
pass
class Ftp(Connection):
"""
:meta private:
Creates a ftp connection instance
@param ip: ip address
@param usr: username
@param pwd: password
@param port: optional, default 21
@return: ftp connection instance
@raise FtpLoginError: raises an exception
"""
def __init__(self, ip, usr, pwd, port=21):
self.ftp = ftplib.FTP()
self.ftp.connect(ip, port)
try:
self.ftp.login(usr, pwd)
except Exception as e:
print("FTP login error: %s" % str(e))
raise Exception("FTP login error: %s" % str(e))
print(self.ftp.getwelcome())
def showFolders(self, path):
return self.ftp.nlst(path)
def createFolder(self, path):
try:
print("Creating folder %s via ftp" % path)
self.ftp.mkd(path)
except Exception as e:
if not "Created directory" in str(e): # dir already created
raise Exception("Error creating dir %s via ftp!!: %s" % (path, str(e)))
def uploadFile(self, originFilePath, destPath):
print("Uploading file %s via ftp" % destPath)
# open file
fileObj = open(originFilePath, "rb")
try:
ftpResponseMessage = self.ftp.storbinary("STOR %s" % destPath, fileObj)
print(ftpResponseMessage)
except Exception as e:
# close file
fileObj.close()
raise Exception("Error uploading %s via ftp!!: %s" % (destPath, str(e)))
# close file
fileObj.close()
def deleteFiles(self, files):
# if only one file convert it to list
if type(files) != list:
files = [files]
for file in files:
try:
self.ftp.delete(file)
print("File %s deleted" % file)
except Exception as e:
if ("Is a directory" or "Directory not empty") in str(e):
# recursive call with all the files inside folder
fileList = [file + "/" + str(x) for x in self.ftp.nlst(file)]
self.deleteFiles(fileList)
try:
self.ftp.rmd(file)
print("Folder %s deleted" % file)
except Exception as e:
print(str(e))
else:
print(str(e))
def exit(self):
self.ftp.quit()
class SFtp(Connection):
"""
:meta private:
Creates a SFtp connection instance
@param ip: ip address
@param user: username
@param key: user private key
@param pwd: optional, default empty
@param port: optional, default 22
@param disAlgorithms: default None
If given, must be a dictionary mapping algorithm type to an iterable of
algorithm identifiers, which will be disabled for the lifetime of the transport.
Example for TSW: dict(pubkeys=['rsa-sha2-256', 'rsa-sha2-512'])
@return: sftp connection instance
@raise sFtpError: raises an exception
"""
def __init__(self, ip, user, key, pwd="", port=22, disAlgorithms=None, timeout=3):
print("SFtp: Connecting to %s as user: %s" % (ip, user))
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
privateKey = paramiko.RSAKey.from_private_key_file(key, pwd)
print("SFtp: Private Key Loaded OK")
self.ssh.connect(ip,
port=port,
username=user,
password=pwd,
pkey=privateKey,
timeout=timeout,
auth_timeout=timeout,
disabled_algorithms=disAlgorithms)
print("SFtp: ssh connection stablished")
self.sftp = self.ssh.open_sftp()
print("SFtp: sftp open")
def showFolders(self, path):
return self.sftp.listdir(path)
def createFolder(self, path):
try:
print("Creating folder %s via sftp" % path)
self.sftp.mkdir(path)
self.sftp.chmod(path, 508) # 508 is 774 in octal
print("Folder %s created via sftp" % path)
except Exception as e:
raise Exception("Error creating dir %s or giving permissions via sftp: %s" % (path, str(e)))
def uploadFile(self, originFilePath, destPath):
print("Uploading file %s via sftp" % destPath)
self.sftp.put(originFilePath, destPath)
# if not "app.bin" in destPath:
# self.sftp.chmod(destPath, 508)
print("File %s uploaded via sftp" % destPath)
def deleteFiles(self, files):
# if only one file convert it to list
if type(files) != list:
files = [files]
for file in files:
# read file info
fileattr = self.__getFileInfoOrDelete(file)
# empty folder or file not exist
if fileattr == False:
continue
# it is a folder
elif S_ISDIR(fileattr.st_mode):
# if is directory make a recursive call
for entry in self.sftp.listdir_attr(file):
self.deleteFiles("%s/%s" % (file, entry.filename))
# after deleting all file delete the folder itself
try:
self.sftp.rmdir(file)
print("Directory %s deleted" % file)
except Exception as e:
print(str(e))
# it is a file
else:
try:
self.sftp.remove(file)
print("File %s deleted" % file)
except Exception as e:
print(str(e))
def __getFileInfoOrDelete(self, file):
'''
Try to get the file information
If not possible check if empty folder
If empty folder delete folder
Else print file not found
'''
try:
return self.sftp.lstat(file)
except Exception as e:
# Could not read file info
print("Check if %s is an empty directory" % file)
try:
self.sftp.rmdir(file)
print("Directory %s deleted" % file)
except Exception as e:
print("File or directory %s does not exist" % file)
return False
def exit(self):
self.sftp.close()
self.ssh.close()