简体字、冯|npm 仓库（nexus/artifactory）批量迁移

背景

公司制品库迁移，要将npm库从原制品库nexus迁移到artifactory中。涉及到近千的依赖包，手动太过费时。因此通过python脚本进行批量操作

思路

梳理思路，单个包的迁移步骤如下：下载包->修改包（package.json）中仓库地址->执行npm publish命令。

与单个包相比，批量操作还需要能拿到包的所有下载地址，在推送之前需要登录npm并设置全局repository地址。

获取包的下载地址

仓库包组件不多，但版本多。因此我选择直接在浏览器中console通过js脚本获取url，如果组件较多建议写脚本获取。

// 将数据以文件形式存储
(function (console) {

    console.save = function (data, filename) {

        if (!data) {

            console.error('Console.save: No data')

            return;

        }

        if (!filename) filename = 'console.json'

        if (typeof data === "object") {

            data = JSON.stringify(data, undefined, 4)

        }

        var blob = new Blob([data], { type: 'text/json' }),

            e = document.createEvent('MouseEvents'),

            a = document.createElement('a')

        a.download = filename

        a.href = window.URL.createObjectURL(blob)

        a.dataset.downloadurl = ['text/json', a.download, a.href].join(':')

        e.initMouseEvent('click', true, false, window, 0, 0, 0, 0, 0, false, false, false, false, 0, null)

        a.dispatchEvent(e)

    }

})(console)
//获取a标签中url
function getHref(a){
    var urlArr = new Array();
    var hrefArr = document.getElementsByTagName('a'); //获取这个页面的所有A标签
    for( var i=0; i<hrefArr.length; i++ ){
        hrefURL = hrefArr[i].href;
        urlArr[i] = hrefURL;
    }
    console.save(urlArr,a);
}
//执行获取脚本
getHref("XXXX.txt");

获取到的url文件（XXXX.txt）如下：

[
    "http://xxxx.com/service/rest/repository/browse/npm/xxxx/",
    "http://xxxx.com/repository/npm/xxxx/xxxx.tgz"
]

python 脚本

代码

replace_file.py

def replace_content(file,old_content,new_content):
    content=read_file(file)
    content=content.replace(old_content,new_content)
    rewrite_file(file,content)
#读文件
def read_file(file):
    with open(file,encoding='UTF-8') as f:
        read_all = f.read()
        f.close()
    return read_all

#写内容到文件
def rewrite_file(file,data):
    with open(file,'w',encoding='UTF-8') as f:
        f.write(data)
        f.close()

auto.py，运行脚本前需要登录有权限的npm账号

import wget
import os
import shutil
import tarfile
import replace_file
import urllib.parse
path='tmp_package/'
utar_path='tmp_utar_package/'
tar_path='tmp_tar_package/'
def tzgName(url):
    #根据url获取压缩包名称
    arr=url.split('-/')
    return arr[1]
def readPackageUrls(f):
    #从文件中读取地址
    urls = []
    f = 'urls/'+f
    file=open(f)
    data_lists=file.readlines()
    for i in data_lists:
        if i.rfind('.tgz') != -1:
            urls.append(i.replace('"','').replace(',\n',''))
    return urls
def dowanloadPackages(urls):
    #下载文件
    for u in urls:
        u=u.strip()
        u=urllib.parse.unquote(u)
        name = path + tzgName(u)
        print(u)
        print(name)
        t=wget.download(u,name)
    return 'download success! size: '
def utar():
    #文件解压
    for f in os.listdir(path):
        dest_dir=f.replace('.tgz','')
        dest_dir=utar_path+dest_dir
        f=path+f
        try:
            tar=tarfile.open(f)
            names=tar.getnames()
            for name in names:
                tar.extract(name,dest_dir)
            tar.close()
        except Exception as e:
            print(e)
def ptar():
    #这个没用
    for f in os.listdir(utar_path):
        tar = tar_path+f+'.tgz'
        f=utar_path+f+'/package/'
        try:
            with tarfile.open(tar,'w:gz') as tar:
                tar.add(f,arcname=os.path.basename(f))
        except Exception as e:
            print(e)          
def repalce():
    #这一步把需要替换的文件中字符串替换掉，主要是仓库地址
    old_path='http://xxxx.com/repository/npm/'
    old_path1='http://xxxx.cn/repository/npm/'
    new_path='https://xxxx.com/artifactory/api/npm/npm-releases/'
    for f in os.listdir(utar_path):
        package_path= utar_path + f + '/package/package.json'
        replace_file.replace_content(package_path,old_path,new_path)
        replace_file.replace_content(package_path,old_path1,new_path)
def dealPackage():
    utar()
    repalce()
    # ptar()
    return 'deal package success!'
def pushPackage():
    for i in os.listdir(utar_path):
        p = utar_path+i+'/package/'
        shell= 'cd ' + p + ' & npm publish & cd ../../../'
        print(shell)
        os.system(shell)
    return 'publish package success!'
def cleanTmp():
    shutil.rmtree(path)
    shutil.rmtree(utar_path)
    shutil.rmtree(tar_path)
    os.makedirs(path)
    os.makedirs(utar_path)
    os.makedirs(tar_path)
cleanTmp()
sum = 0
for f in os.listdir('urls'):
    us = readPackageUrls(f)
    cleanTmp()
    dowan=dowanloadPackages(us)
    deal=dealPackage()
    push=pushPackage()
    print('处理文件：%s，共:%d条数据'%(f,len(us)))
    sum+=len(us)
    print('共处理文件：%d 条数据'%(sum))

简体字、冯|npm 仓库（nexus/artifactory）批量迁移

背景

思路

获取包的下载地址

python 脚本

目录

代码