CDDIS

[AdrianKriger]

I've raised this question on StackExchange

CDDIS now requires authentication.
I am attempting to hack the download.py with utility functions courtesy pyTMD.

import os
import sys
import ssl
import base64
import posixpath

from urllib.parse import quote_plus
from http.cookiejar import CookieJar
import urllib.request as url

#-- PURPOSE: "login" to NASA Earthdata with supplied credentials
def build_opener(username, password, context=ssl.SSLContext(ssl.PROTOCOL_TLS),
    password_manager=True, get_ca_certs=True, redirect=True,
    authorization_header=False, urs='https://urs.earthdata.nasa.gov'):
    """
    build urllib opener for NASA Earthdata with supplied credentials
    Parameters
    ----------
    username: str or NoneType, default None
        NASA Earthdata username
    password: str or NoneType, default None
        NASA Earthdata password
    context: obj, default ssl.SSLContext()
        SSL context for url opener object
    password_manager: bool, default True
        Create password manager context using default realm
    get_ca_certs: bool, default True
        Get list of loaded “certification authority” certificates
    redirect: bool, default True
        Create redirect handler object
    authorization_header: bool, default False
        Add base64 encoded authorization header to opener
    urs: str, default 'https://urs.earthdata.nasa.gov'
        Earthdata login URS 3 host
    """
    #-- https://docs.python.org/3/howto/urllib2.html#id5
    handler = []
    #-- create a password manager
    if password_manager:
        password_mgr = url.HTTPPasswordMgrWithDefaultRealm()
        #-- Add the username and password for NASA Earthdata Login system
        password_mgr.add_password(None, urs, username, password)
        handler.append(url.HTTPBasicAuthHandler(password_mgr))
    #-- Create cookie jar for storing cookies. This is used to store and return
    #-- the session cookie given to use by the data server (otherwise will just
    #-- keep sending us back to Earthdata Login to authenticate).
    cookie_jar = CookieJar()
    handler.append(url.HTTPCookieProcessor(cookie_jar))
    #-- SSL context handler
    if get_ca_certs:
        context.get_ca_certs()
    handler.append(url.HTTPSHandler(context=context))
    #-- redirect handler
    if redirect:
        handler.append(url.HTTPRedirectHandler())
    #-- create "opener" (OpenerDirector instance)
    opener = url.build_opener(*handler)
    #-- Encode username/password for request authorization headers
    #-- add Authorization header to opener
    if authorization_header:
        b64 = base64.b64encode('{0}:{1}'.format(username, password).encode())
        opener.addheaders = [("Authorization","Basic {0}".format(b64.decode()))]
    #-- Now all calls to urllib2.urlopen use our opener.
    url.install_opener(opener)
    #-- All calls to urllib2.urlopen will now use handler
    #-- Make sure not to include the protocol in with the URL, or
    #-- HTTPPasswordMgrWithDefaultRealm will be confused.
    return opener

#-- PURPOSE: check that entered NASA Earthdata credentials are valid
def check_credentials():
    """
    Check that entered NASA Earthdata credentials are valid
    """
    try:
        remote_path = posixpath.join('https://cddis.nasa.gov','archive')
        request = url.Request(url=remote_path)
        response = url.urlopen(request, timeout=20)
    except url.HTTPError:
        raise RuntimeError('Check your NASA Earthdata credentials')
    except url.URLError:
        raise RuntimeError('Check internet connection')
    else:
        return True

Then we can change the GNSSpy function so:

def get_sp3(sp3file, directory=os.getcwd()):
    """
    This function downloads IGS orbit file from NASA CDDIS ftp server.
    """
    fileName = sp3file + ".Z"
    if os.path.exists(fileName) == True:
        if os.path.exists(fileName[:-2]) == True:
            print(fileName[-2] + " exists in working directory")
            return
        else:
            print(fileName + " exists in working directory | Extracting...")
            Archive(fileName + ".Z").extractall(os.getcwd())
            return
    internet = check_internet()
    if internet == False:
        raise Warning('No internet connection! | Cannot download orbit file')
        
    username = 'someusername'
    password = 'somepassword'
    
     #-- build opener
    build_opener(username, password)
    #-- check credentials
    check_credentials()
     
    server  = 'https://cddis.nasa.gov/archive/'
    sp3FileDir = 'gnss/products'
    if sp3file.startswith("wum"): 
        sp3FileDir += '/mgex'
    file_topath = os.path.join(directory, fileName)
    fileDir = [server, sp3FileDir, fileName[3:-7], fileName]
    site= '/'.join(fileDir) # FTP link of file

    try:
        print('Downloading:', fileName, end = '')
        
        request = url.Request(site)
        
        #url.urlretrieve(ftp, file_topath)
        
        # Create an http response object
        with url.urlopen(request) as response:
        # Create a file object
            with open(file_topath, "wb") as f:
            # Copy the binary content of the response to the file
                shutil.copyfileobj(response, f)
        
        print(' | Download completed for', fileName)
        Archive(fileName).extractall(os.getcwd())
    except:
        print(" | Requested file", fileName, "cannot be not found!")

When I follow the instructions and execute:

import gnsspy as gp
stn = gp.read_obsFile('./log_rinex/Ctwn-SB_obs_log_202111081430.obs')

./log_rinex/Ctwn-SB_obs_log_202111081430.obs exist in working directory | Reading... Observation file ./log_rinex/Ctwn-SB_obs_log_202111081430.obs is read in 6.46 seconds.
and:

orbit = gp.sp3_interp(stn.epoch, interval=stn.interval, sp3_product="igs", clock_product="igs")

I get:

Downloading: igs21830.sp3.Z | Download completed for igs21830.sp3.Z
 | Requested file igs21830.sp3.Z cannot be found!
patool: Extracting igs21830.sp3.Z ...
patool: running C:\Users\miniconda3\envs\rt_ppp-env\Library\bin\7z.EXE e -oC:\rtklib_realTime_PPP\log_rinex -- igs21830.sp3.Z
Traceback (most recent call last):

  Input In [104] in <cell line: 1>
    orbit = gp.sp3_interp(stn.epoch, interval=stn.interval, sp3_product="igs", clock_product="igs")

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\gnsspy\position\interpolation.py:23 in sp3_interp
    yes   = readFile.read_sp3File(yesterday)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\gnsspy\io\readFile.py:569 in read_sp3File
    isexist(sp3file)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\gnsspy\funcs\checkif.py:86 in isexist
    extract_archive(fileName + ".Z", outdir=_CWD)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\patoolib\__init__.py:684 in extract_archive
    return _extract_archive(archive, verbosity=verbosity, interactive=interactive, outdir=outdir, program=program)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\patoolib\__init__.py:484 in _extract_archive
    run_archive_cmdlist(cmdlist, verbosity=verbosity)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\patoolib\__init__.py:421 in run_archive_cmdlist
    return util.run_checked(cmdlist, verbosity=verbosity, **runkwargs)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\patoolib\util.py:227 in run_checked
    raise PatoolError(msg)

PatoolError: Command `['C:\\Users\\miniconda3\\envs\\rt_ppp-env\\Library\\bin\\7z.EXE', 'e', '-oC:\\rtklib_realTime_PPP\\log_rinex', '--', 'igs21830.sp3.Z']' returned non-zero exit status 2

The .sp3.7 has a file size of 13KB. It should be 96KB.

[AdrianKriger]

Went with the CDDIS instruction.

.netrc in the HOME folder (Users/username) then;

 # Makes request of URL, stores response in variable r
 r = requests.get(site, timeout=20)
 # Opens a local file of same name as remote file for writing to
  with open(file_topath, 'wb') as fd:
         for chunk in r.iter_content(chunk_size=1000):
               fd.write(chunk)
 # Closes local file
 fd.close()

But now:

import gnsspy as gp
stn = gp.read_obsFile('Ctwn-SB_obs_log_202111081430.obs')

Ctwn-SB_obs_log_202111081430.obs exist in working directory | Reading... Observation file Ctwn-SB_obs_log_202111081430.obs is read in 2.76 seconds.

then:

orbit = gp.sp3_interp(stn.epoch, interval=stn.interval, sp3_product="gfz", clock_product="gfz")

Downloading: gfz21830.sp3.Zpatool: Extracting gfz21830.sp3.Z ...
patool: running C:\Users\miniconda3\envs\rt_ppp-env\Library\bin\7z.EXE e -oC:\rtklib_realTime_PPP\log_rinex -- gfz21830.sp3.Z
patool: ... gfz21830.sp3.Z extracted to `C:\rtklib_realTime_PPP\log_rinex'.
gfz21830.sp3 file is read in 16.93 seconds
Downloading: gfz21831.sp3.Zpatool: Extracting gfz21831.sp3.Z ...
patool: running C:\miniconda3\envs\rt_ppp-env\Library\bin\7z.EXE e -oC:\rtklib_realTime_PPP\log_rinex -- gfz21831.sp3.Z
patool: ... gfz21831.sp3.Z extracted to `C:\rtklib_realTime_PPP\log_rinex'.
gfz21831.sp3 file is read in 11.42 seconds
Downloading: gfz21832.sp3.Zpatool: Extracting gfz21832.sp3.Z ...
patool: running C:\Users\miniconda3\envs\rt_ppp-env\Library\bin\7z.EXE e -oC:\rtklib_realTime_PPP\log_rinex -- gfz21832.sp3.Z
patool: ... gfz21832.sp3.Z extracted to `C:\rtklib_realTime_PPP\log_rinex'.
gfz21832.sp3 file is read in 14.77 seconds
Downloading: cod21831.clk_05s.Z | Download completed for cod21831.clk_05s.Z
patool: Extracting cod21831.clk_05s.Z ...
patool: running C:\Users\miniconda3\envs\rt_ppp-env\Library\bin\7z.EXE e -oC:\rtklib_realTime_PPP\log_rinex -- cod21831.clk_05s.Z
patool: ... cod21831.clk_05s.Z extracted to `C:\rtklib_realTime_PPP\log_rinex'.
cod21831.clk_05s file is read in 18.68 seconds
Traceback (most recent call last):

  Input In [15] in <cell line: 1>
    orbit = gp.sp3_interp(stn.epoch, interval=stn.interval, sp3_product="gfz", clock_product="gfz")

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\gnsspy\position\interpolation.py:85 in sp3_interp
    epoch_interp_List[:,:,svIndex] = sv_interp

ValueError: could not broadcast input array from shape (10800,6) into shape (360,6)

With sp3_product="igs", clock_product="igs" same error.
Have I missed something?

[AdrianKriger]

I eventually got this to execute successfully with:
orbit = gp.sp3_interp(stn.epoch, sp3_product="gfz", clock_product="gfz").

Subsequently its all been:

Downloading: {any-file-here}.sp3.Z | Requested file {any-file-here}.sp3.Z cannot be found!
An exception has occurred, use %tb to see the full traceback.

Traceback (most recent call last):

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\urllib3\response.py:441 in _error_catcher
    yield

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\urllib3\response.py:522 in read
    data = self._fp.read(amt) if not fp_closed else b""

  File ~\miniconda3\envs\rt_ppp-env\lib\http\client.py:459 in read
    n = self.readinto(b)

  File ~\miniconda3\envs\rt_ppp-env\lib\http\client.py:503 in readinto
    n = self.fp.readinto(b)

  File ~\miniconda3\envs\rt_ppp-env\lib\socket.py:669 in readinto
    return self._sock.recv_into(b)

  File ~\miniconda3\envs\rt_ppp-env\lib\ssl.py:1241 in recv_into
    return self.read(nbytes, buffer)

  File ~\miniconda3\envs\rt_ppp-env\lib\ssl.py:1099 in read
    return self._sslobj.read(len, buffer)

ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host

During handling of the above exception, another exception occurred:

Traceback (most recent call last):

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\models.py:760 in generate
    for chunk in self.raw.stream(chunk_size, decode_content=True):

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\urllib3\response.py:579 in stream
    data = self.read(amt=amt, decode_content=decode_content)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\urllib3\response.py:544 in read
    raise IncompleteRead(self._fp_bytes_read, self.length_remaining)

  File ~\miniconda3\envs\rt_ppp-env\lib\contextlib.py:131 in __exit__
    self.gen.throw(type, value, traceback)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\urllib3\response.py:458 in _error_catcher
    raise ProtocolError("Connection broken: %r" % e, e)

ProtocolError: ("Connection broken: ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)", ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\gnsspy\download.py:380 in get_sp3
    r = requests.get(address, timeout=20)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\api.py:75 in get
    return request('get', url, params=params, **kwargs)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\api.py:61 in request
    return session.request(method=method, url=url, **kwargs)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\sessions.py:529 in request
    resp = self.send(prep, **send_kwargs)

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\sessions.py:667 in send
    history = [resp for resp in gen]

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\sessions.py:667 in <listcomp>
    history = [resp for resp in gen]

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\sessions.py:237 in resolve_redirects
    resp = self.send(

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\sessions.py:687 in send
    r.content

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\models.py:838 in content
    self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''

  File ~\miniconda3\envs\rt_ppp-env\lib\site-packages\requests\models.py:763 in generate
    raise ChunkedEncodingError(e)

ChunkedEncodingError: ("Connection broken: ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)", ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))

During handling of the above exception, another exception occurred:

SystemExit
C:\Users\miniconda3\envs\rt_ppp-env\lib\site-packages\IPython\core\interactiveshell.py:3405: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.
  warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)

I haven't been able to access CDDIS anymore.
Is this usual behaviour?
Perhaps a Discussion` where the community share their experience?

[MrBourriz]

How did u get the skyplots, for me it didn't work

[AdrianKriger]

I went a completely different route.

Harvest 'Dilution of Precision' and 'Azimuth' from rtklib and then generate a skyplot.

[MrBourriz]

I think you can generate directly a skyplot using rtklib.
I was looking for an other way, I found a new python package https://github.com/geumjin99/PyRINEX and also a code using the output of teqc http://www.westernexplorers.us/GNSSplotters/