Module cvtool.core.miptables

This file is set up such that importing it will run the relevant setup scripts.


WARN: This does mean that once imported, it will not import again unless you start a new independent program.


A module that manages the following: - checks for existing versions of the mip tables - downloading the latest version of the mip tables - saves these to the system temporary directory.

Requires: pip install GitPython

Todo: - Replace print statements with logger

Expand source code
'''
This file is set up such that importing it will run the relevant setup scripts.
*******
WARN: This does mean that once imported, it will not import again unless you start a new independent program.
*******

A module that manages the following:
 - checks for existing versions of the mip tables
 - downloading the latest version of the mip tables
 - saves these to the system temporary directory.

Requires: pip install GitPython

Todo:
- Replace print statements with logger

'''

from typing import List, Dict, Any
import git
from git import Repo
import tempfile
import glob
import os,json
from . import io
from .version_control import last_commit, query_repo, clear_last  # Assuming version_control is properly set up

__REPOPREFIX__ = 'cvtool.miptables.'

def setup_mip_tables(commit_hash = None) -> None:
    """
    Set up the MIP tables by checking for existing versions, downloading the latest version, and updating environment variables.
    """



    repo_url = 'https://github.com/PCMDI/mip-cmor-tables'


    # print('WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN - using wolfiex (test miptables) instead of PCMDI repo')
    # print('WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN - using wolfiex (test miptables) instead of PCMDI repo')
    # repo_url = 'https://github.com/wolfiex/mip-cmor-tables'

    table_subdir = ''

    term = io.terminal()
    print('*'*term.columns)

    if not commit_hash:
        # Get the last update from the GitHub repository
        current = last_commit(*repo_url.split('/')[-2:])
    else:
        current = dict(SHA=commit_hash)
        print(f'User specified MIPTABLE version')

    # The location of the system temporary files
    tmp = tempfile.gettempdir()
    repo = None

    # Check if we have any existing repositories in the tmp
    existing = glob.glob(f'{tmp}/{__REPOPREFIX__}*')


    for path in existing:
        t_repo = Repo(path)
        if str(t_repo.head.commit) == current.get('SHA') and t_repo.remotes.origin.url == repo_url:
            # Set the main repo as the existing one
            repo = t_repo
            existing.remove(path)

            # Remove the outdated duplicates if they exist
            for p in existing:
                io.rmdir(p)
            break

    if not repo:
        # If the repo does not exist or is not current, we download a new one.
        try:
            temp_dir = tempfile.mkdtemp(prefix=__REPOPREFIX__, suffix='')
        except FileExistsError:
            print("Directory already exists with the same name [this is unusual]")

        # Let's download the latest repository
        repo = Repo.clone_from(repo_url + '.git', temp_dir)

        # if we have specified a commit hash 
        if commit_hash:
            try:
                # Check if the commit exists
                commit = repo.commit(commit_hash)
                # Revert to the specified commit (creates a new commit)
                repo.git.reset('--hard', commit)

                print(f"Reverted to commit {commit_hash}")
            except git.exc.GitCommandError:
                print(f"Commit {commit_hash} not found in the repository.")
                print('WARN - using the latest version of the miptables instead.')

        if str(repo.head.commit) != current.get('SHA'):
            user = input(f"\n\nCommit messages: {str(repo.head.commit)} != {current.get('SHA')} \n This is likely because the MIPTable repository has been updated recently.\n\nType 'yes', to update the repository info and try again. \n\n")
            if user == 'yes':
                last = repo_url.split('/')[-2:]
                clear_last(*last)
                current = last_commit(*last)
            else:
                try:
                    # we revert to the last saved commit. 
                    commit_hash = current.get('SHA')
                    commit = repo.commit(commit_hash)
                    repo.git.reset('--hard', commit)
                    print(f"Reverted to commit {commit_hash}")
                except git.exc.GitCommandError:
                    print(f"Commit {commit_hash} not found in the repository.")


        assert str(repo.head.commit) == current.get('SHA'), 'Commit hashses must match. [core->MipTables]'

    # Get the repository URL
    repo_url = repo.remotes.origin.url
    # Get the current commit hash
    current_commit_hash = repo.head.commit.hexsha
    # Create clickable links
    repo_link = repo_url.replace(".git", "").replace(":", "/").replace("git@", "https://")
    commit_link = f"{repo_link}/commit/{current_commit_hash}"

    LOCATION = repo.working_dir + table_subdir
    # os.environ['cmor_tables'] = LOCATION
    # os.environ['cmor_tableorigin'] = commit_link

    miptables = query_repo(repo.working_dir,verbose=False)
    info = query_repo(repo.working_dir,verbose=True)
    
    print('*'*term.columns)
    print(f"""
          MipTable location: {LOCATION} 
          With Commit: {current.get('SHA')}
          
          Latest Tag: {info['tag']['latest']}
          Latest Commit:{info['commit']['latest']}
          
          Viewable URL: {commit_link}

          """)
    print('*'*term.columns)


    institutions = json.load(open(f'{LOCATION}/MIP_institutions.json','r'))
    return miptables, LOCATION, institutions

Functions

def setup_mip_tables(commit_hash=None) ‑> None

Set up the MIP tables by checking for existing versions, downloading the latest version, and updating environment variables.

Expand source code
def setup_mip_tables(commit_hash = None) -> None:
    """
    Set up the MIP tables by checking for existing versions, downloading the latest version, and updating environment variables.
    """



    repo_url = 'https://github.com/PCMDI/mip-cmor-tables'


    # print('WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN - using wolfiex (test miptables) instead of PCMDI repo')
    # print('WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN WARN - using wolfiex (test miptables) instead of PCMDI repo')
    # repo_url = 'https://github.com/wolfiex/mip-cmor-tables'

    table_subdir = ''

    term = io.terminal()
    print('*'*term.columns)

    if not commit_hash:
        # Get the last update from the GitHub repository
        current = last_commit(*repo_url.split('/')[-2:])
    else:
        current = dict(SHA=commit_hash)
        print(f'User specified MIPTABLE version')

    # The location of the system temporary files
    tmp = tempfile.gettempdir()
    repo = None

    # Check if we have any existing repositories in the tmp
    existing = glob.glob(f'{tmp}/{__REPOPREFIX__}*')


    for path in existing:
        t_repo = Repo(path)
        if str(t_repo.head.commit) == current.get('SHA') and t_repo.remotes.origin.url == repo_url:
            # Set the main repo as the existing one
            repo = t_repo
            existing.remove(path)

            # Remove the outdated duplicates if they exist
            for p in existing:
                io.rmdir(p)
            break

    if not repo:
        # If the repo does not exist or is not current, we download a new one.
        try:
            temp_dir = tempfile.mkdtemp(prefix=__REPOPREFIX__, suffix='')
        except FileExistsError:
            print("Directory already exists with the same name [this is unusual]")

        # Let's download the latest repository
        repo = Repo.clone_from(repo_url + '.git', temp_dir)

        # if we have specified a commit hash 
        if commit_hash:
            try:
                # Check if the commit exists
                commit = repo.commit(commit_hash)
                # Revert to the specified commit (creates a new commit)
                repo.git.reset('--hard', commit)

                print(f"Reverted to commit {commit_hash}")
            except git.exc.GitCommandError:
                print(f"Commit {commit_hash} not found in the repository.")
                print('WARN - using the latest version of the miptables instead.')

        if str(repo.head.commit) != current.get('SHA'):
            user = input(f"\n\nCommit messages: {str(repo.head.commit)} != {current.get('SHA')} \n This is likely because the MIPTable repository has been updated recently.\n\nType 'yes', to update the repository info and try again. \n\n")
            if user == 'yes':
                last = repo_url.split('/')[-2:]
                clear_last(*last)
                current = last_commit(*last)
            else:
                try:
                    # we revert to the last saved commit. 
                    commit_hash = current.get('SHA')
                    commit = repo.commit(commit_hash)
                    repo.git.reset('--hard', commit)
                    print(f"Reverted to commit {commit_hash}")
                except git.exc.GitCommandError:
                    print(f"Commit {commit_hash} not found in the repository.")


        assert str(repo.head.commit) == current.get('SHA'), 'Commit hashses must match. [core->MipTables]'

    # Get the repository URL
    repo_url = repo.remotes.origin.url
    # Get the current commit hash
    current_commit_hash = repo.head.commit.hexsha
    # Create clickable links
    repo_link = repo_url.replace(".git", "").replace(":", "/").replace("git@", "https://")
    commit_link = f"{repo_link}/commit/{current_commit_hash}"

    LOCATION = repo.working_dir + table_subdir
    # os.environ['cmor_tables'] = LOCATION
    # os.environ['cmor_tableorigin'] = commit_link

    miptables = query_repo(repo.working_dir,verbose=False)
    info = query_repo(repo.working_dir,verbose=True)
    
    print('*'*term.columns)
    print(f"""
          MipTable location: {LOCATION} 
          With Commit: {current.get('SHA')}
          
          Latest Tag: {info['tag']['latest']}
          Latest Commit:{info['commit']['latest']}
          
          Viewable URL: {commit_link}

          """)
    print('*'*term.columns)


    institutions = json.load(open(f'{LOCATION}/MIP_institutions.json','r'))
    return miptables, LOCATION, institutions