由浅入深 SCF 无服务器云函数实践

# -*- coding: utf-8 -*-
from ctypes import *
import os
import base64
import json
import calculate_biz_host_num

def main_handler(event, context):
    return calculate_biz_host_num.main()

if __name__ == '__main__':
    '''just for test'''
    event = {}
    lambda_handler(event, 4)

# -*- coding: utf-8 -*-
from ctypes import *
import os
import pictype
import base64
import json

def main_handler(event, context):
    str = pictype.cppmain(event["pic_data"])
    jso = json.loads(str)
    print jso["QRCode"]
    return str

if __name__ == '__main__':
    '''just for test'''
    event = {}
    imageFile = open("2qrcode.jpg","rb")
    event["pic_data"] = base64.b64encode(imageFile.read())
    imageFile.close()
    lambda_handler(event, 4)

# -*- coding utf-8 -*-
import os
import sys
import datetime
import traceback
import shutil
import commands
import cos_sdk


def main_handler(event, context):
    res = map_caller(event, context)
    if res == 0:
        return "succ"
    else:
        return "fail"

def map_caller(event, context):
    # Note: this is test account, change to own cos appid and secret_id
    appid = '123443xxxx'
    secret_id = 'QmFzZTY0IGlzIGEgZ2Vxxxx'
    secret_key = 'AKIDZfbOA78asKUYBcXFrJD0a1ICvxxxx'
    host = 'sz.cxxxxxx'
    addr = '10xxxx'

    bucket = event['bucket']
    cos_input_file = event['input']
    cos_output_key = event['output']
    cos_file_name = cos_input_file.split('/', 1)[1]
    print("cos_file_name: ", cos_file_name)

    # step 1. Download .abs file from cos
    cos = cos_sdk.CosHandler(appid, bucket, secret_id, secret_key, host, addr )
    container_base_path = '/tmp/AITest/mapper'
    container_input_path = '/tmp/AITest/mapper/input/'
    container_output_path = '/tmp/AITest/mapper/output/'
    cos_output_path = 'mapdata/'

    try:
        if not os.path.exists(container_base_path):
            os.makedirs(container_base_path)
        if not os.path.exists(container_input_path):
            os.mkdir(container_input_path)
        if not os.path.exists(container_output_path):
            os.mkdir(container_output_path)
    except:
        traceback.print_exc()
        return -1

    ret = cos.download_file('/', cos_input_file, container_input_path, cos_file_name)
    if not ret:
        print("Download file from cos Failed [%s]" % cos_file_name)
        return -1
    print("Download file [%s] succ" % cos_file_name)

    # step 2. transfer .abs file to .hdf5
    ret = transfer_data(container_base_path, container_input_path, cos_file_name, container_output_path)
    if not ret:
        print("transfer data fail")
        return -1

    # step 3. upload .hdf5 file to cos
    output_filename = get_output_file(cos_file_name, container_output_path)
    if output_filename == "":
        return -1
    print(container_output_path+output_filename)
    if not os.path.exists(container_output_path+output_filename):
        print "output file not exist"
    print(cos_output_path+output_filename)
    ret = cos.upload_file(container_output_path+output_filename, cos_output_path+output_filename)
    if not ret:
        return -1

    ## clean up result files
    shutil.rmtree(container_output_path)
    return 0

def transfer_data(base_path, file_path, file_name, output_path):
    try:
        CurPath = '/var/user'
        if os.path.exists(base_path + '/transfer_script'):
            shutil.rmtree(base_path + '/transfer_script')
        shutil.copytree(CurPath + '/transfer_script', base_path + '/transfer_script')
        StartPath = base_path + "/transfer_script/5v5_vecmodel_tactics"
        InputFilePath = file_path + file_name
        OutputPath = output_pat
        sgameBinFile = base_path + "/transfer_script/log_transform/bin/transform/sgame_log_transform"
        os.chmod(sgameBinFile, 755)
        labelBinFile = StartPath + "/label"
        featureBinFile = StartPath + "/VecFeatureExtract"
        os.chmod(labelBinFile, 755)
        os.chmod(featureBinFile, 755)
        os.chdir(StartPath)
        cmd = 'sh start.sh ' + InputFilePath + ' ' + OutputPath
        (status, output) = commands.getstatusoutput(cmd)
        print("status: ", status)
        print("output: ", output)
    except:
        traceback.print_exc()
        return False

    if status == 0 and output == "pipeline success":
        return True
    else:
        return False

# -*- coding utf-8 -*-
import os
import sys
import traceback
import shutil
import commands
import re
import common
import cos_sdk

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET


def main_handler(event, context):
    res = reducer_caller(event, context)
    if res == False:
        return "fail"
    else:
        return "succ"

def reducer_caller(event, context):
    # Note: this is test account, change to own cos appid and secret_id
    appid = '12344321xxx'
    secret_id = 'QmFzZTY0IGlzIGEgZ2xxxx'
    secret_key = 'AKIDZfbOA78asKUYBcXFrJD0axxx'
    host = 'sz.xxxx'
    addr = '10.xxxx'

    bucket = 'mapreduce'
    container_base_path = '/tmp/AITest/reducer'
    container_mapfile_path = '/tmp/AITest/reducer/mapdata/'
    container_output_path = '/tmp/AITest/reducer/output/'
    cos_mapdata_dir = '/'
    cos_output_key = u'output/'

    ## AI shuffle config
    p0_thread_num = '30'
    p1_thread_num = '4'
    ai_bucket = '4'
    sample_num = '5120'

    ## init container directory
    try:
        if not os.path.exists(container_base_path):
            os.makedirs(container_base_path)
        if not os.path.exists(container_mapfile_path):
            os.mkdir(container_mapfile_path)
        if not os.path.exists(container_output_path):
            os.mkdir(container_output_path)
    except:
        traceback.print_exc()
        return False

    cos = cos_sdk.CosHandler(appid, bucket, secret_id, secret_key, host, addr)

    ## step 1. get all mapper output data name (*.abs)
    min_mapfiles = 40
    mapfiles = get_mapfiles(cos, bucket)
    if mapfiles == []:
        print("No exist data map file in cos, please run lambda mapper first")
        return False
    elif len(mapfiles) < min_mapfiles:
        print("No enough map files in cos, at least %d map files can trigger shuffle process" % min_mapfiles)
        return False

    ## step 2. download mapper data from cos
    for mapfile in mapfiles:
        download_ret = download_file(cos, cos_mapdata_dir, mapfile, container_mapfile_path)
        if download_ret != 0:
            return False   

    # step 3. shuffle mapper input file
    ret = shuffle_data(container_base_path, container_mapfile_path, container_output_path, p0_thread_num, p1_thread_num, ai_bucket, sample_num)
    if not ret:
        print("shuffle data fail")
        return False

    # step 4. upload .hdf5 file to cos
    output_files = get_output_files(container_output_path)
    if len(output_files) == 0:
        print("No output results in *.hdf5")
        return False

    all_upload_ret = 0
    print(output_files)
    for output_file in output_files:
        upload_ret = upload_file(cos, container_output_path, output_file, cos_output_key)
        all_upload_ret += upload_ret
        if upload_ret != 0:
            print("Upload output file [%s] to cos failed" % output_file)
    if all_upload_ret != 0:
        return False

    ## clean up result files
    shutil.rmtree(container_output_path)
    return "Finish shuffle data"


def get_mapfiles(cos_client, bucket):
    status, ret_msg = cos_client.list_object(bucket)
    if str(status)[0] != '2':
        print("Get map data file error")
        return -1

    mapfiles = []
    root = ET.fromstring(ret_msg)
    for key in root.findall('Contents'):
        filename = key.find('Key').text
        if re.match(r'^mapdata/', filename):
            mapfiles.append(filename)
    return mapfiles

def shuffle_data(container_base_path, container_mapfile_path, container_output_path, p0_thread_num, p1_thread_num, ai_bucket, sample_num):
    try:
        CurPath = '/var/user'
        shuffle_tools = '/shuffle_all_tools/'
        if os.path.exists(container_base_path + shuffle_tools):
            shutil.rmtree(container_base_path + shuffle_tools)
        shutil.copytree(CurPath + shuffle_tools, container_base_path + shuffle_tools)

        os.chdir(container_base_path + shuffle_tools)
        cmd = 'sh king_shuffle_start.sh ' + container_mapfile_path + ' ' + p0_thread_num + ' ' + p1_thread_num + ' ' + container_output_path + ' ' + ai_bucket + ' ' + sample_num
        print cmd
        val = os.system(cmd)
        print val
        return True
    except:
        traceback.print_exc()
        return False