技术标签： perl
下面的Perl脚本为武汉大学测绘学院李星星教授开源的linux端GNSS数据下载脚本，为了方便使用，本人编写了Python脚本用于数据分类、解压缩和重命名等操作
使用方式为：./download_files.py -year -doy -timeLen -fileType -agency
1、下载o文件：download_files.py 2020 130 1 obs
2、下载轨道文件：download_files.py 2020 130 1 sp3 igs
3、下载钟差文件：download_files.py 2020 130 1 clk igs
4、下载DCB文件：download_files.py 2020 130 1 dcb
5、下载广播星历文件：download_files.py 2020 130 1 nav
6、下载snx文件：download_files.py 2020 130 1 snx
7、下载eop文件：download_files.py 2020 130 1 eop
curl_gnss.pl
#!/usr/bin/perl
##
# Usage : IGS/MGEX data/products downloading
# Created : Yuan Yongqiang, 2020/10/24
# Updated : Yuan Yongqiang, 2020/10/25, adding EOP and PRODUCTS downloading
# Anonymous，2021/2/15, the gound sites selection function is added
#
# -obs daily daily_v2 daily_v3 hourly hourly_v2 hourly_v3
# -nav brdc hourly daily
# -dcb cas code (dlr not supported now)
# -snx igs
# -orb gfz wum grg cod igs
# -clk gfz wum grg cod igs
# -eop all daily gpsrapid
# -slr all
#
## 1. Get a list of all files in a directory:
# curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/*?list"
#
## 2. Get a list of files in a directory matching a pattern (cs2rx1700*.001.Z)
# curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700*.001.Z?list"
#
## 3. Download a group of files within a range
# curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700[1-9].001.Z" -O
#
## 4. Download a group of files from a list of specific files
# curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700{1,3,7}.001.Z" -O
#
## 5. Download a set of files matching a pattern and output those to a single .tar file
# curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700*.001.Z" -o files.tar
#
## 6. Download a single file
# curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx17001.001.Z" -O
#
use strict;
use warnings;
use Net::FTP;
use Getopt::Long;
use File::Copy;
use File::Path qw( make_path );
use Data::Dumper;
use File::Basename;
use Class::Struct;
use Cwd;
use threads;
use threads::shared;
use Time::Local;
struct DataType => {
type => '$', ntype => '$',
time => '$', ctypes => '@',
};
struct IpMode => {
ip => '$', dir => '$',
mode => '$', type => '$',
subtype => '$',
};
struct CurlNode => {
ip => '$', user => '$',
pwd => '$', dir => '$',
file => '$', save => '$',
};
my $year=0;
my $doy = 0;
my $hour = 0;
my $nthreads = 3;
my $save = "";
my @obs = ();
my @nav = ();
my @dcb = ();
my @snx = ();
my @orb = ();
my @clk = ();
my @eop = ();
my @slr = ();
GetOptions (
"year=i" => \$year,
"doy=i" => \$doy,
"hour=i" => \$hour,
"nth=i" => \$nthreads,
"save=s" => \$save,
"obs=s{,}" => \@obs,
"nav=s{,}" => \@nav,
"dcb=s{,}" => \@dcb,
"snx=s{,}" => \@snx,
"orb=s{,}" => \@orb,
"clk=s{,}" => \@clk,
"eop=s{,}" => \@eop,
"slr=s{,}" => \@slr,
);
if($nthreads > 10) {
print "Max threads 10\n";
exit;
}
my @DataTypes = ();
my $ctime = sprintf("%04d%03d%02d",$year,$doy,$hour);
push @DataTypes,DataType->new(time=>$ctime, type=>"obs", ntype=>$#obs+1, ctypes=>\@obs) if(@obs);
push @DataTypes,DataType->new(time=>$ctime, type=>"nav", ntype=>$#nav+1, ctypes=>\@nav) if(@nav);
push @DataTypes,DataType->new(time=>$ctime, type=>"dcb", ntype=>$#dcb+1, ctypes=>\@dcb) if(@dcb);
push @DataTypes,DataType->new(time=>$ctime, type=>"snx", ntype=>$#snx+1, ctypes=>\@snx) if(@snx);
push @DataTypes,DataType->new(time=>$ctime, type=>"orb", ntype=>$#orb+1, ctypes=>\@orb) if(@orb);
push @DataTypes,DataType->new(time=>$ctime, type=>"clk", ntype=>$#clk+1, ctypes=>\@clk) if(@clk);
push @DataTypes,DataType->new(time=>$ctime, type=>"eop", ntype=>$#eop+1, ctypes=>\@eop) if(@eop);
push @DataTypes,DataType->new(time=>$ctime, type=>"slr", ntype=>$#slr+1, ctypes=>\@slr) if(@slr);
my @requested_CurlNodes = make_download_list(@DataTypes,$save);
if(@requested_CurlNodes and ($#requested_CurlNodes+1) >= 1) {
curl_download_files(@requested_CurlNodes,$nthreads);
}
exit;
sub curl_download_files {
my (@requested_CurlNodes) = @_;
my $nthreads = pop(@requested_CurlNodes);
my @cur_download_cmds = ();
foreach my $requested_CurlNode (@requested_CurlNodes) {
my $ip = $requested_CurlNode->ip;
my $user = $requested_CurlNode->user;
my $pwd = $requested_CurlNode->pwd;
my $rmtdir = $requested_CurlNode->dir;
my $file = $requested_CurlNode->file;
my $savedir = $requested_CurlNode->save;
system("mkdir -p $savedir") if(! -d $savedir);
my $cmd_curl = "";
if($ip =~ /cddis/ or $ip =~ /http/) {
$cmd_curl = "curl -c .urs_cookies -b .urs_cookies -n -L \"$ip$rmtdir/$file\" -o $savedir/$file";
}
else {
$cmd_curl = "curl $ip$rmtdir/$file -u anonymous:yqyuanwhu --silent -o $savedir/$file";
}
if(! -e "$savedir/$file") {
push @cur_download_cmds,$cmd_curl;
}
else {
print "Omitting already existed file: $savedir/$file\n";
}
}
my @threads = ();
my $nfiles_per_thread = int(($#cur_download_cmds+1)/$nthreads);
$nthreads = 1 if($nfiles_per_thread<2);
#print "$nthreads\n";
#print "$nfiles_per_thread\n";
#exit;
for(my $ith = 0; $ith < $nthreads; $ith ++) {
my $istart = $ith*$nfiles_per_thread;
my $iend = ($ith+1)*($nfiles_per_thread)-1;
if($ith == $nthreads - 1) {
$iend = $#cur_download_cmds;
}
#print "$istart $iend\n";
my @th_cur_download_cmds = ();
for(my $icmd = $istart; $icmd <= $iend; $icmd ++) {
push @th_cur_download_cmds, $cur_download_cmds[$icmd];
}
if(@th_cur_download_cmds) {
my $th_cur = threads->create(\&run_download_files, @th_cur_download_cmds);
push @threads,$th_cur;
}
}
foreach my $curl_download_thread (@threads) {
$curl_download_thread->join();
}
}
sub run_download_files {
my (@cur_download_cmds) = @_;
foreach my $cur_download_cmd (@cur_download_cmds) {
#print "$cur_download_cmd\n";
my $time1=timelocal(localtime());
system($cur_download_cmd);
my $time2=timelocal(localtime());
my $diff=$time2-$time1;
print "Time consuming: $diff seconds; Finished: $cur_download_cmd\n";
}
}
sub make_download_list {
my (@DataTypes) = @_;
my $save = pop(@DataTypes);
my @IpModes = ();
foreach my $DT (@DataTypes) {
my $year = substr($DT->time,0,4);
my $doy = substr($DT->time,4,3);
my $hour = substr($DT->time,7,2);
my $type = $DT->type;
if($DT->ntype > 0) {
foreach my $subtype (@{$DT->ctypes}) {
push @IpModes,resolve_ip($year,$doy,$hour,$type,$subtype);
}
}
}
my @requested_CurlNodes = ();
foreach my $ipnode (@IpModes) {
my $ip = $ipnode->ip;
my $dir = $ipnode->dir;
my $mode = $ipnode->mode;
my $type = $ipnode->type;
my $subtype = $ipnode->subtype;
print "Requesting files on remote server: ip = $ip, dir = $dir, mode = $mode, type = $type, subtype = $subtype\n";
push @requested_CurlNodes, get_http_remote_list($year,$doy,$hour,$type,$subtype,$ip,$dir,$mode,$save);
}
return @requested_CurlNodes;
}
sub resolve_ip {
my ($year,$doy,$hour,$type,$subtype) = @_;
my $ip_cddis = "https://cddis.nasa.gov";
my $ip_ign = "ftp://igs.ign.fr";
my $ip_whu = "ftp://igs.gnsswhu.cn";
my $ip_aiub = "ftp://ftp.aiub.unibe.ch";
my $ip_tum = "ftp://edc.dgfi.tum.de";
my $ip_iers = "ftp://ftp.iers.org";
my $mjd = yeardoy2mjd($year,$doy);
my ($gwk,$gwkd) = mjd2gwkd($mjd);
my ($year_not_used, $month, $monthday) = doy2monthday($year, $doy);
my @IpModes = ();
# -obs cddis/ign/whu houly houly_v2 houly_v3 daily daily_v2 daily_v3
# -nav cddis/ign/whu brdc hourly daily
# -dcb cddis/aiub cas dlr code
# -snx cddis/ign/whu igs
# -orb cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -clk cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -eop cddis/iers all daily gpsrapid
# -slr cddis/tum all prn
#### OBSERVATIONS
if($type eq "obs") {
my $mode_v2 = "*d.gz";
my $mode_v3 = "*MO.crx.gz";
my $dir_cddis = "";
my $dir_ign = "";
my $dir_whu = "";
if($subtype =~ /hourly/) {
$dir_cddis = sprintf("/archive/gnss/data/hourly/%04d/%03d/%02d",$year,$doy,$hour);
$dir_ign = sprintf("/pub/igs/data/hourly/%04d/%03d",$year,$doy);
$dir_whu = sprintf("/pub/gps/data/hourly/%04d/%03d/%02d",$year,$doy,$hour);
}
if($subtype =~ /daily/) {
$dir_cddis = sprintf("/archive/gnss/data/daily/%04d/%03d/%02dd",$year,$doy,substr($year,2,2));
$dir_ign = sprintf("/pub/igs/data/%04d/%03d",$year,$doy);
$dir_whu = sprintf("/pub/gps/data/daily/%04d/%03d/%02dd",$year,$doy,substr($year,2,2));
}
if($subtype =~ /_v2/ or $subtype =~ /_v3/) {
if($subtype =~ /_v2/) {
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$mode_v2, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_ign, dir=>$dir_ign, mode=>$mode_v2, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_whu, dir=>$dir_whu, mode=>$mode_v2, type=>$type, subtype=>$subtype);
}
if($subtype =~ /_v3/) {
open(FILE,"<","/home/huangshi/project/software_hs/scripts/downloadscript_perl/site_list")||die"cannot open the file: $!\n";
my @linelist=<FILE>;
my $eachline='';
foreach $eachline(@linelist){
chomp $eachline;
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$eachline.$mode_v3, type=>$type, subtype=>$subtype);
}
close FILE;
#push @IpModes,IpMode->new(ip=>$ip_ign, dir=>$dir_ign, mode=>$mode_v3, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_whu, dir=>$dir_whu, mode=>$mode_v3, type=>$type, subtype=>$subtype);
}
}
else {
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$mode_v2, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_ign, dir=>$dir_ign, mode=>$mode_v2, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_whu, dir=>$dir_whu, mode=>$mode_v2, type=>$type, subtype=>$subtype);
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$mode_v3, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_ign, dir=>$dir_ign, mode=>$mode_v3, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_whu, dir=>$dir_whu, mode=>$mode_v3, type=>$type, subtype=>$subtype);
}
}
#### NAVIGATION
if($type eq "nav") {
if($subtype =~ /brdc/) {
my $dir = sprintf("/archive/gnss/data/daily/%04d/brdc",$year);
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>sprintf("*%02dn.gz",substr($year,2,2)), type=>$type, subtype=>$subtype); # GPS
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>sprintf("*%02dg.gz",substr($year,2,2)), type=>$type, subtype=>$subtype); # GLO
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>"BRD*MN.rnx.gz", type=>$type, subtype=>$subtype); # Multi-GNSS
$dir = sprintf("/archive/gnss/data/daily/%04d/%03d/%02dp/",$year,$doy,substr($year,2,2));
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>"BRD*MN.rnx.gz", type=>$type, subtype=>$subtype); # Multi-GNSS
}
if($subtype =~ /hourly/) {
my $dir = sprintf("/archive/gnss/data/hourly/%04d/%03d/%02d",$year,$doy,$hour);
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>"*N.rnx.gz", type=>$type, subtype=>$subtype); # Longname-only
}
if($subtype =~ /daily/) {
my $dir = sprintf("/archive/gnss/data/daily/%04d/%03d/%02dp",$year,$doy,substr($year,2,2));
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>"*N.rnx.gz", type=>$type, subtype=>$subtype); # Longname-only
}
}
#### DCB
if($type eq "dcb") {
if($subtype =~ /cod/) {
my $dir = sprintf("/CODE/%04d",$year);
my $mode = "*.DCB.Z";
push @IpModes,IpMode->new(ip=>$ip_aiub, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
if($subtype =~ /cas/) {
my $dir = sprintf("/archive/gnss/products/bias/%04d",$year);
my $mode = "*.BSX.gz";
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
if($subtype =~ /dlr/) {
# to be added
}
}
#### SINEX
if($type eq "snx") {
my $dir = sprintf("/archive/gnss/products/%04d",$gwk);
my $mode = sprintf("igs*%04d.snx.Z",$gwk);
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
$mode = sprintf("igs*%04d.snx.gz",$gwk);
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
#### ORBIT/SP3
if($type eq "orb") {
if($subtype =~ /igs/) {
my $dir = sprintf("/archive/gnss/products/%04d",$gwk);
my $mode = "igs*.sp3.Z";
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
$mode = "igs*.sp3.gz";
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
else {
my $dir = sprintf("/archive/gnss/products/mgex/%04d",$gwk);
my $mode = sprintf("%3s0MGXFIN*ORB.SP3.gz",uc($subtype));
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
$mode = sprintf("%3s0MGXRAP*ORB.SP3.gz",uc($subtype));
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
}
#### CLOCK
if($type eq "clk") {
if($subtype =~ /igs/) {
my $dir = sprintf("/archive/gnss/products/%04d",$gwk);
my $mode = "igs*.clk_30s.Z";
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
$mode = "igs*.clk_30s.gz";
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
else {
my $dir = sprintf("/archive/gnss/products/mgex/%04d",$gwk);
my $mode = sprintf("%3s0MGXFIN*CLK.CLK.gz",uc($subtype));
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
$mode = sprintf("%3s0MGXRAP*CLK.CLK.gz",uc($subtype));
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
}
#### EOP
if($type eq "eop") {
my $dir = "/archive/products/iers";
my $mode = "";
if($subtype =~ /daily/) {
$mode = "*2000A.daily";
#$dir = "/products/eop/rapid/daily";
}
if($subtype =~ /all/) {
$mode = "*2000A.data";
#$dir = "/products/eop/rapid/standard";
}
if($subtype =~ /gpsrapid/) {
$mode = "gpsrapid*";
#$dir = "/products/eop/rapid/standard";
}
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
#push @IpModes,IpMode->new(ip=>$ip_iers, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
#### SLR
if($type eq "slr") {
my @slr_sats = ("galileo101","galileo102","galileo103","galileo104","galileo201","galileo202","galileo203","galileo204",
"galileo205","galileo206","galileo207","galileo208","galileo209","galileo210","galileo211","galileo212",
"galileo213","galileo214","galileo215","galileo216","galileo217","galileo218","galileo219","galileo220",
"galileo221","galileo222","qzs1","qzs2","qzs3","qzs4","beidou3m2","beidou3m3","beidou3m9","beidou3m10",
"compassg1","compassi3","compassi5","compassi6b","compassm3");
my $mode = sprintf("%04d%02d.npt",$year,$month);
foreach my $slr_sat (@slr_sats) {
my $dir = sprintf("/archive/slr/data/npt_crd/%s/%04d",$slr_sat,$year);
push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
## for TUM ftp
#my $dir = sprintf("/pub/slr/data/npt_crd/%s/%04d",$slr_sat,$year);
#push @IpModes,IpMode->new(ip=>$ip_tum, dir=>$dir, mode=>$mode, type=>$type, subtype=>$subtype);
}
}
return @IpModes;
}
sub get_http_remote_list {
my ($year,$doy,$hour,$type,$subtype,$ip,$dir,$mode,$save) = @_;
my @requested_CurlNode = ();
## use http for cddis
if($ip =~ /cddis/) {
my @remote_lists = `curl -c .urs_cookies -b .urs_cookies -n -L --silent "$ip$dir/$mode?list"`;
my $total = `curl -c .urs_cookies -b .urs_cookies -n -L --silent "$ip$dir/$mode?list" | grep 'Total number of files'`;
print $total;
if(! $total) {
print "No files!\n";
return @requested_CurlNode;
#exit;
}
else {
foreach my $remote_list (@remote_lists) {
my @token=split(/ /,$remote_list);
my $filename = $token[0];
if(looks_like_gnss($year,$doy,$hour,$type,$subtype,$filename)) {
my $node = CurlNode->new(ip=>$ip,user=>"yqyuan",pwd=>"yqyuanwhu",dir=>$dir,file=>$filename,save=>$save);
push @requested_CurlNode, $node;
}
}
}
}
## use ftp for others
else {
my @remote_lists = `curl -u anonymous:yqyuanwhu --silent -l "$ip$dir/$mode"`;
if(! @remote_lists) {
print "No files!\n";
return @requested_CurlNode;
#exit;
}
else {
foreach my $remote_list (@remote_lists) {
my $filename = $remote_list;
chomp($filename);
if(looks_like_gnss($year,$doy,$hour,$type,$subtype,$filename)) {
#print "$filename";
my $node = CurlNode->new(ip=>$ip,user=>"yqyuan",pwd=>"yqyuanwhu",dir=>$dir,file=>$filename,save=>$save);
push @requested_CurlNode, $node;
}
}
}
}
return @requested_CurlNode;
}
sub looks_like_gnss {
my ($year,$doy,$hour,$type,$subtype,$remote_name) = @_;
my $mjd = yeardoy2mjd($year,$doy);
my ($gwk,$gwkd) = mjd2gwkd($mjd);
my ($year_not_used, $month, $monthday) = doy2monthday($year, $doy);
my @hour_id_basket = ("a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x");
my $hour_id = $hour_id_basket[$hour];
# -obs cddis/ign/whu houly houly_v2 houly_v3 daily daily_v2 daily_v3
# -nav cddis/ign/whu brdc hourly daily
# -dcb cddis/aiub cas dlr code
# -snx cddis/ign/whu igs
# -orb cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -clk cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -eop cddis/iers all daily gpsrapid
# -slr cddis/tum all prn
my @ext_modes = ();
if($type =~ /obs/) {
push @ext_modes,sprintf("%03d0.%02dd.Z",$doy,substr($year,2,2));
push @ext_modes,sprintf("%03d%s.%02dd.Z",$doy,$hour_id,substr($year,2,2));
push @ext_modes,sprintf("%03d0.%02dd.gz",$doy,substr($year,2,2));
push @ext_modes,sprintf("%03d%s.%02dd.gz",$doy,$hour_id,substr($year,2,2));
push @ext_modes,sprintf("%04d%03d0000_01D_30S_MO.crx.gz",$year,$doy);
push @ext_modes,sprintf("%04d%03d%02d00_01H_30S_MO.crx.gz",$year,$doy,$hour);
}
if($type =~ /nav/) {
#push @ext_modes,sprintf("%04d%03d%02d00_01H_GN.rnx.gz",$year,$doy,$hour);
#push @ext_modes,sprintf("%04d%03d%02d00_01H_RN.rnx.gz",$year,$doy,$hour);
#push @ext_modes,sprintf("%04d%03d%02d00_01H_EN.rnx.gz",$year,$doy,$hour);
#push @ext_modes,sprintf("%04d%03d%02d00_01H_CN.rnx.gz",$year,$doy,$hour);
#push @ext_modes,sprintf("%04d%03d%02d00_01H_JN.rnx.gz",$year,$doy,$hour);
push @ext_modes,sprintf("%04d%03d%02d00_01H_MN.rnx.gz",$year,$doy,$hour);
#push @ext_modes,sprintf("%03d%s.%02dn.Z",$doy,$hour_id,substr($year,2,2));
#push @ext_modes,sprintf("%03d%s.%02dg.Z",$doy,$hour_id,substr($year,2,2));
#push @ext_modes,sprintf("%03d%s.%02dl.Z",$doy,$hour_id,substr($year,2,2));
#push @ext_modes,sprintf("%03d%s.%02df.Z",$doy,$hour_id,substr($year,2,2));
#push @ext_modes,sprintf("%03d%s.%02dq.Z",$doy,$hour_id,substr($year,2,2));
#push @ext_modes,sprintf("%03d%s.%02dp.Z",$doy,$hour_id,substr($year,2,2));
push @ext_modes,sprintf("%04d%03d0000_01D_MN.rnx.gz",$year,$doy);
push @ext_modes,sprintf("%03d0.%02dp.Z",$doy,substr($year,2,2));
push @ext_modes,sprintf("%03d0.%02dp.gz",$doy,substr($year,2,2));
}
if($type =~ /dcb/) {
push @ext_modes,sprintf("%04d%03d0000_01D_01D_DCB.BSX.gz",$year,$doy);
push @ext_modes,sprintf("P1C1%02d%02d.DCB.Z",substr($year,2,2),$month);
push @ext_modes,sprintf("P1P2%02d%02d.DCB.Z",substr($year,2,2),$month);
push @ext_modes,sprintf("P2C2%02d%02d_RINEX.DCB.Z",substr($year,2,2),$month);
}
if($type =~ /snx/) {
push @ext_modes,sprintf("%04d.snx.Z",$gwk);
push @ext_modes,sprintf("%04d.snx.gz",$gwk);
}
if($type =~ /orb/) {
push @ext_modes,sprintf("igs%04d%01d.sp3.Z",$gwk,$gwkd);
push @ext_modes,sprintf("igs%04d%01d.sp3.gz",$gwk,$gwkd);
push @ext_modes,sprintf("%04d%03d0000_01D_05M_ORB.SP3.gz",$year,$doy);
push @ext_modes,sprintf("%04d%03d0000_01D_15M_ORB.SP3.gz",$year,$doy);
}
if($type =~ /clk/) {
push @ext_modes,sprintf("igs%04d%01d.clk_30s.Z",$gwk,$gwkd);
push @ext_modes,sprintf("igs%04d%01d.clk_30s.gz",$gwk,$gwkd);
push @ext_modes,sprintf("%04d%03d0000_01D_30S_CLK.CLK.gz",$year,$doy);
}
if($type =~ /eop/) {
push @ext_modes,"finals2000A.data";
push @ext_modes,"finals2000A.daily";
push @ext_modes,"gpsrapid.out";
push @ext_modes,"gpsrapid.daily";
}
if($type =~ /slr/) {
push @ext_modes,sprintf("%04d%02d.npt",$year,$month);
}
my $lgnss = 0;
foreach my $ext_mode (@ext_modes) {
if($remote_name =~ /$ext_mode/) {
$lgnss = 1;
last;
}
}
return $lgnss;
}
sub yeardoy2mjd {
my ($iyear, $idoy) = @_;
my $iyear2 = $iyear -1;
my $mjd;
{
use integer;
$mjd = 365*$iyear-678941+$iyear2/4-$iyear2/100+$iyear2/400;
}
$mjd = $mjd + $idoy;
return $mjd;
}
sub mjd2gwkd {
my ($mjd) = @_;
my $gwk = int(($mjd - 44244)/7);
my $gwkd = $mjd-44244 - $gwk * 7;
return ($gwk,$gwkd);
}
sub doy2monthday {
my @days_in_month = (31,28,31,30,31,30,31,31,30,31,30,31);
my ($iyear,$idoy) = @_;
$days_in_month[1]= ($iyear%4==0 && $iyear%100!=0 || $iyear%400 == 0) ? 29 : 28;
my $iday = $idoy;
my $imonth;
for($imonth=0;$imonth<12;$imonth++) {
if($iday <= $days_in_month[$imonth]) {
last;
}
$iday = $iday - $days_in_month[$imonth];
}
$imonth++;
return (sprintf("%04d",$iyear),sprintf("%02d",$imonth),sprintf("%02d",$iday));
}
download_files.py
#!/home/anonymous/project/InstallSoft/Python/bin/python3.7
"""
Usage : IGS/MGEX data/products downloading
Created : Anonymous, Feb.15,2021
"""
import sys
import os
import glog
import datetime
def yeardoy2mjd(iyear,idoy):
iyear2=iyear-1
mjd=365*iyear-678941+int(iyear2/4)-int(iyear2/100)+int(iyear2/400)
mjd=mjd+idoy
return mjd
def mjd2gwkd(mjd):
gwk=int((mjd-44244)/7)
gwkd=mjd-44244-gwk*7
return str(gwk)+str(gwkd)
if (len(sys.argv)<2):
glog.fatal('Usage: '+sys.argv[0]+' -year'+' -doy'+' -timeLen'+' -fileType'+' -agency')
sys.exit()
glog.info('start downloading files!')
start_time=datetime.datetime.now()
glog.info('the executed file name is:'+sys.argv[0])
script_name=sys.argv[0]
glog.info('1st arg year is:'+sys.argv[1])
year=sys.argv[1]
glog.info('2nd arg beg doy is:'+sys.argv[2])
doy0=int(sys.argv[2])
glog.info('3rd arg time lenth is:'+sys.argv[3])
time_len=int(sys.argv[3])
glog.info('4th arg file type is:'+sys.argv[4])
file_type=sys.argv[4]
if (len(sys.argv)>5):
glog.info('5th arg agency is:'+sys.argv[5])
agency=sys.argv[5]
crt_path=os.getcwd()
main_perl=os.path.join(crt_path,'downloadscript_perl','curl_gnss.pl')
os.system(f"chmod +x {main_perl}")
glog.info(main_perl)
for i in range(time_len):
doy=doy0+i
cdoy=str(doy).zfill(3)
glog.info(str(year)+str(cdoy))
#BRDM files
if (file_type=="nav"):
nav_dir=os.path.join('/home/anonymous/scratch/data/nav',year,cdoy)
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {nav_dir} -nav brdc")
os.system(f"cd {nav_dir}")
os.system(f"uncompress {nav_dir}/*")
os.system(f"mv {nav_dir}/BRDM* {nav_dir}/brdm{cdoy}0.{year[-2:]}p")
#Observation files
if (file_type=="obs"):
obs_dir=os.path.join('/home/anonymous/scratch/data/obs',year,cdoy)
crx2rnx="/home/anonymous/project/software_hs/tool/crx2rnx"
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {obs_dir} -obs daily_v3 --nth 10")
os.system(f"cd {obs_dir}")
os.system(f"uncompress {obs_dir}/*")
for root,dirs,files in os.walk(obs_dir):
for file in files:
if (os.path.splitext(file)[1]=='.crx'):
os.system(f"{crx2rnx} {obs_dir}/{file} -f")
os.system(f"rm {obs_dir}/{file}")
site_upper=file[0:4]
site_lower=site_upper.lower()
os.system(f"mv {obs_dir}/{site_upper}*.rnx {obs_dir}/{site_lower}{cdoy}0.{year[-2:]}o")
#DCB files
if (file_type=="dcb"):
dcb_dir=os.path.join('/home/anonymous/scratch/data/dcb',year)
os.system(f"mkdir -p {dcb_dir}")
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {dcb_dir} -dcb cas -nth 10")
os.system(f"uncompress {dcb_dir}/*.gz")
#SNX files
if (file_type=="snx"):
snx_dir=os.path.join('/home/anonymous/scratch/data/snx_hs',year)
if (i%7!=0): continue
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {snx_dir} -snx igs -nth 10")
os.system(f"uncompress {snx_dir}/*.Z")
#SP3 files gfz/wum/grg/cod/igs
if (file_type=="sp3"):
sp3_dir=os.path.join('/home/anonymous/scratch/data/sp3',year,cdoy)
if (agency=="igs"):
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {sp3_dir} -orb igs -nth 10")
os.system(f"uncompress {sp3_dir}/*.Z")
continue
elif (agency in ["gfz","wum","grg","cod"]):
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {sp3_dir} -orb {agency} -nth 10")
else:
glog.info('ONLY SUPPORT GFZ/WUM/GRG/COD/IGS FOR SP3')
break
mjd=yeardoy2mjd(int(year),int(cdoy))
gwkd=mjd2gwkd(mjd)
os.system(f"uncompress {sp3_dir}/*.gz")
os.system(f"mv {sp3_dir}/{agency.upper()}*.SP3 {sp3_dir}/{agency}{gwkd}.sp3")
#Clock offset files
if (file_type=="clk"):
clk_dir=os.path.join('/home/anonymous/scratch/data/clk',year,cdoy)
mjd=yeardoy2mjd(int(year),int(cdoy))
gwkd=mjd2gwkd(mjd)
if (agency=="igs"):
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {clk_dir} -clk igs -nth 10")
os.system(f"uncompress {clk_dir}/*.Z")
os.system(f"mv {clk_dir}/{agency}*.clk* {clk_dir}/{agency}{gwkd}.clk")
continue
elif (agency in ["gfz","wum","grg","cod"]):
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {clk_dir} -clk {agency} -nth 10")
else:
glog.info('ONLY SUPPORT GFZ/WUM/GRG/COD/IGS FOR CLK')
break
os.system(f"uncompress {clk_dir}/*.gz")
os.system(f"mv {clk_dir}/{agency.upper()}*.CLK {clk_dir}/{agency}{gwkd}.clk")
#SLR npt files
if (file_type=="slr"):
slr_dir=os.path.join('/home/anonymous/scratch/data/slr',year)
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {slr_dir} -slr all -nth 10")
# os.system(f"uncompress {slr_dir}/*.Z")
#EOP files
if (file_type=="eop"):
eop_dir=os.path.join('/home/anonymous/scratch/data/eop',year,cdoy)
os.system(f"perl {main_perl} --year {year} --doy {cdoy} --save {eop_dir} -eop daily -nth 10")
finish_time=datetime.datetime.now()
time_cost=finish_time-start_time
glog.info('start time is: '+str(start_time))
glog.info('finish time is: '+str(finish_time))
glog.info('time cost is: '+str(time_cost))
文章浏览阅读331次。第一部分：准备工作1 安装虚拟机2 安装centos73 安装JDK以上三步是准备工作，至此已经完成一台已安装JDK的主机第二部分：准备３台虚拟机以下所有工作最好都在root权限下操作1 克隆上面已经有一台虚拟机了,现在对master进行克隆,克隆出另外2台子机;1.1 进行克隆21.2 下一步1.3 下一步1.4 下一步1.5 根据子机需要,命名和安装路径1.6 ..._创建一个hadoop项目
文章浏览阅读1.7k次。心脏滴血漏洞HeartBleed CVE-2014-0160 是由heartbeat功能引入的，本文从深入码层面的分析该漏洞产生的原因_heartbleed代码分析
文章浏览阅读1.4k次。前言ofd是国家文档标准，其对标的文档格式是pdf。ofd文档是容器格式文件，ofd其实就是压缩包。将ofd文件后缀改为.zip，解压后可看到文件包含的内容。ofd文件分析工具下载：点我下载。ofd文件解压后，可以看到如下内容： 对于xml文件，可以用文本工具查看。但是对于印章文件(Seal.esl)、签名文件(SignedValue.dat)就无法查看其内容了。本人开发一款ofd内容查看器，..._signedvalue.dat
文章浏览阅读1.8w次，点赞29次，收藏313次。整体系统设计本设计主要是对ADC和DAC的使用，主要实现功能流程为：首先通过串口向FPGA发送控制信号，控制DAC芯片tlv5618进行DA装换，转换的数据存在ROM中，转换开始时读取ROM中数据进行读取转换。其次用按键控制adc128s052进行模数转换100次，模数转换数据存储到FIFO中，再从FIFO中读取数据通过串口输出显示在pc上。其整体系统框图如下：图1：FPGA数据采集系统框图从图中可以看出，该系统主要包括9个模块：串口接收模块、按键消抖模块、按键控制模块、ROM模块、D.._基于fpga的信息采集
文章浏览阅读2.5w次。1.背景错误信息：-- [http-nio-9904-exec-5] o.s.c.n.z.filters.post.SendErrorFilter : Error during filteringcom.netflix.zuul.exception.ZuulException: Forwarding error at org.springframework.cloud..._com.netflix.zuul.exception.zuulexception
文章浏览阅读358次。1.介绍图的相关概念 图是由顶点的有穷非空集和一个描述顶点之间关系-边（或者弧）的集合组成。通常，图中的数据元素被称为顶点，顶点间的关系用边表示，图通常用字母G表示，图的顶点通常用字母V表示，所以图可以定义为: G=(V,E)其中，V(G)是图中顶点的有穷非空集合，E(G)是V(G)中顶点的边的有穷集合1.1 无向图：图中任意两个顶点构成的边是没有方向的1.2 有向图：图中..._给定一个邻接矩阵未必能够造出一个图
文章浏览阅读321次。（十二）、WDS服务器安装通过前面的测试我们会发现，每次安装的时候需要加域光盘映像，这是一个比较麻烦的事情，试想一个上万个的公司，你天天带着一个光盘与光驱去给别人装系统，这将是一个多么痛苦的事情啊，有什么方法可以解决这个问题了？答案是肯定的，下面我们就来简单说一下。WDS服务器，它是Windows自带的一个免费的基于系统本身角色的一个功能，它主要提供一种简单、安全的通过网络快速、远程将Window..._doc server2012上通过wds+mdt无人值守部署win11系统.doc
文章浏览阅读219次。python–xlrd/xlwt/xlutilsxlrd只能读取，不能改,支持 xlsx和xls 格式xlwt只能改，不能读xlwt只能保存为.xls格式xlutils能将xlrd.Book转为xlwt.Workbook，从而得以在现有xls的基础上修改数据，并创建一个新的xls，实现修改xlrd打开文件import xlrdexcel=xlrd.open_workbook('E:/test.xlsx') 返回值为xlrd.book.Book对象,不能修改获取sheett_xlutils模块可以读xlsx吗
文章浏览阅读8.2w次，点赞267次，收藏656次。运行Selenium出现'WebDriver' object has no attribute 'find_element_by_id'或AttributeError: 'WebDriver' object has no attribute 'find_element_by_xpath'等定位元素代码错误，是因为selenium更新到了新的版本，以前的一些语法经过改动。..............._unresolved attribute reference 'find_element_by_id' for class 'webdriver
文章浏览阅读198次。一：模态窗口//父页面JSwindow.showModalDialog(ifrmehref, window, 'dialogWidth:550px;dialogHeight:150px;help:no;resizable:no;status:no');//子页面获取父页面DOM对象//window.showModalDialog的DOM对象var v=parentWin..._jquery获取父window下的dom对象
文章浏览阅读1.7w次，点赞15次，收藏129次。算法（algorithm）是解决一系列问题的清晰指令，也就是，能对一定规范的输入，在有限的时间内获得所要求的输出。 简单来说，算法就是解决一个问题的具体方法和步骤。算法是程序的灵 魂。二、算法的特征1.可行性 算法中执行的任何计算步骤都可以分解为基本可执行的操作步，即每个计算步都可以在有限时间里完成（也称之为有效性） 算法的每一步都要有确切的意义，不能有二义性。例如“增加x的值”，并没有说增加多少，计算机就无法执行明确的运算。 _算法
文章浏览阅读1.5k次，点赞18次，收藏26次。网络安全的标准和规范是网络安全领域的重要组成部分。它们为网络安全提供了技术依据，规定了网络安全的技术要求和操作方式，帮助我们构建安全的网络环境。下面，我们将详细介绍一些主要的网络安全标准和规范，以及它们在实际操作中的应用。_网络安全标准规范