WIP workflow [skip actions]

.github/workflows/BuildTest.yml

@@ -1,10 +1,19 @@
 # This is a basic workflow to help you get started with Actions
 name: Manually executable test for ESP-IDF v4.3.1
-on: workflow_dispatch
+on: 
-# on:
+  workflow_dispatch:
-#   push:
+    inputs:
-#     branches: 
+          ui_build:
-#       - '**4.3'
+            description: 'Rebuild UI'
+            required: true
+            type: boolean
+          release_build:
+            description: 'Release'
+            required: true
+            type: boolean
+  push:
+    branches: 
+      - '**4.3'
 
 jobs:
   bootstrap:
@@ -31,12 +40,14 @@ jobs:
       id: build_flags
       run: |
         git config --global --add safe.directory /__w/squeezelite-esp32/squeezelite-esp32   
+        echo "Dumping environment"
+        env
         . /opt/esp/python_env/idf4.3_py3.8_env/bin/activate
         # build_flags support the following options
         # --mock - to mock the compilation part - this is to be used for testing only
         # --force - to force a release build even if the last commit message doesn't contain the word "release"
         # --ui_build - to force a ui_build even if the last commit message doesn't contain "[ui-build]"
-        build_tools.py build_flags  --ui_build
+        build_tools.py build_flags 
 
   preparation:
     name: Preparation
@@ -51,15 +62,11 @@ jobs:
           echo "Web Build Flag=${{needs.bootstrap.outputs.ui_build}}"
           echo "Mock flag=${{needs.bootstrap.outputs.mock}}"
           echo "Release Flag=${{needs.bootstrap.outputs.release_flag}}"      
+          git config --global --add safe.directory /__w/squeezelite-esp32/squeezelite-esp32   
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
         submodules: true
-    - name: Pull latest 
-      run: |
-        git config --global --add safe.directory /__w/squeezelite-esp32/squeezelite-esp32   
-        git pull 
-        git submodule update
     - name: Refresh certificates
       run: |
         git update-index --chmod=+x ./server_certs/getcert.sh
@@ -105,12 +112,9 @@ jobs:
       image: sle118/squeezelite-esp32-idfv43
     needs: [preparation,bootstrap]
     strategy:
-      max-parallel: 3
       matrix:
-        #node: [I2S-4MFlash, SqueezeAmp, Muse]
+        node: [I2S-4MFlash, SqueezeAmp, Muse]
-        #depth: [16, 32]
+        depth: [16, 32]
-        node: [I2S-4MFlash]
-        depth: [16]
         exclude:
           - node: Muse
             depth: 32      

.gitignore

@@ -16,5 +16,5 @@ components/wifi-manager/UML-State-Machine-in-C
 *.bak
 
 envfile.txt
-
+artifacts
 web-installer

docker/build_tools.py

@@ -160,6 +160,12 @@ parser_build_flags.add_argument('--mock', action='store_true',help='Mock release
 parser_build_flags.add_argument('--force', action='store_true',help='Force a release build')
 parser_build_flags.add_argument('--ui_build', action='store_true',help='Include building the web UI')
 
+def format_commit(commit):
+  #463a9d8b7 Merge branch 'bugfix/ci_deploy_tags_v4.0' into 'release/v4.0' (2020-01-11T14:08:55+08:00)
+  dt = datetime.fromtimestamp(float(commit.author.time), timezone( timedelta(minutes=commit.author.offset) ))
+  timestr = dt.strftime('%c%z')
+  cmesg= commit.message.replace('\n', ' ' )
+  return f'{commit.short_id} {cmesg} ({timestr}) <{commit.author.name}>'.replace('  ', ' ', )
 
 def get_github_data(repo:Repository,api):
     base_url = urlparse(repo.remotes['origin'].url)
@@ -312,6 +318,7 @@ class Releases():
   maxcount:int =0
   branch:str=''
   repo:Repository=None
+  last_commit:Commit = None
   manifest_name:str
   def __init__(self,branch:str,maxcount:int=3) -> None:
     self.maxcount = maxcount
@@ -358,20 +365,27 @@ class Releases():
     elif package.has_artifacts or not with_artifacts:
       self.append(package)
   @classmethod
+  def get_last_commit_message(cls)->str:
+      last:Commit = cls.get_last_commit()
+      if last is None:
+        return ''
+      else:
+        return last.message.replace('\n', ' ')
+  @classmethod
   def get_last_commit(cls)->Commit:
     if cls.repo is None:
       cls.get_repository(os.getcwd())
-      target=cls.repo.head.target
+      head:Reference  = cls.repo.head
-
+      target=head.target
-      last_commit=''
+      ref:Reference
-      try:
+      if cls.last_commit is None:
-        last_commit=cls.repo[last_commit]
+        try:
-        logger.info(f'Last commit for target {target} is {last_commit}')
+          cls.last_commit=cls.repo[target]
-      except Exception as e:
+          logger.info(f'Last commit for {head.shorthand} is {format_commit(cls.last_commit)}')
-        logger.error(f'Unable to retrieve last commit for target {target}: {e}')
+        except Exception as e:
-        last_commit=None
+          print(f'::error::Unable to retrieve last commit for {head.shorthand}/{target}: {e}')
-      
+          cls.last_commit=None
-    return last_commit
+    return cls.last_commit
   @classmethod
   def get_repository(cls,path:str=os.getcwd())->Repository:
     if cls.repo is None:  
@@ -379,9 +393,9 @@ class Releases():
         logger.info(f'Opening repository from {path}')
         cls.repo=Repository(path=path)
       except GitError as ex:
-        print(f'::error::Error while trying to access the repository.')
+        print(f'::error::Unable to access the repository.')
         print(f'::error::Content of {path}:')
-        print('\n::error::'.join(get_file_list(path)))
+        print('\n::error::'.join(get_file_list(path,1)))
         raise 
     return cls.repo
   @classmethod
@@ -428,6 +442,27 @@ class Releases():
         break
 
     return packages
+  @classmethod 
+  def get_commit_list(cls)->list:
+    commit_list = []
+    last:Commit = Releases.get_last_commit()
+    if last is None:
+      return commit_list
+    try:
+      for c in Releases.get_repository().walk(last.id,pygit2.GIT_SORT_TIME):
+        commit_list.append(format_commit(c))
+        if len(commit_list)>10:
+            break
+
+    except Exception as e:
+      print(f'::error::Unable to get commit list starting at {last.id}: {e}')
+
+    return commit_list
+
+  @classmethod
+  def get_commit_list_descriptions(cls)->str:
+    return '<<~EOD\n### Revision Log<br>\n'+'<br>\n'.join(cls.get_commit_list())+'\n~EOD'
+    
   def update(self, *args, **kwargs):
       if args:
           if len(args) > 1:
@@ -466,8 +501,8 @@ def write_github_env(args):
   logger.info(f'Writing environment details to {args.env_file}...')
   with open(args.env_file,  "w") as env_file:
     for attr in [attr for attr in dir(github_env) if not attr.startswith('_')]:
-      line=f'{attr}={getattr(github_env,attr)}'
+      line=f'{attr}{"=" if attr != "description" else ""}{getattr(github_env,attr)}'
-      logger.info(line)
+      print(line)
       env_file.write(f'{line}\n')
       os.environ[attr] = str(getattr(github_env,attr))
   logger.info(f'Done writing environment details to {args.env_file}!')
@@ -479,12 +514,6 @@ def set_workflow_output(args):
     os.environ[attr] = str(getattr(github_env,attr))
   logger.info(f'Done outputting job variables!')  
 
-def format_commit(commit):
-  #463a9d8b7 Merge branch 'bugfix/ci_deploy_tags_v4.0' into 'release/v4.0' (2020-01-11T14:08:55+08:00)
-  dt = datetime.fromtimestamp(float(commit.author.time), timezone( timedelta(minutes=commit.author.offset) ))
-  timestr = dt.strftime('%c%z')
-  cmesg= commit.message.replace('\n', ' ' )
-  return f'{commit.short_id} {cmesg} ({timestr}) <{commit.author.name}>'.replace('  ', ' ', )
 
 def format_artifact_name(base_name:str='',args = AttributeDict(os.environ)):
   return f'{base_name}{args.branch_name}-{args.node}-{args.depth}-{args.major}{args.build}'
@@ -493,7 +522,7 @@ def handle_build_flags(args):
   set_workdir(args)
   logger.info('Setting global build flags')
   last:Commit = Releases.get_last_commit()
-  commit_message:str= last.message.replace('\n', ' ')
+  commit_message:str= Releases.get_last_commit_message()
   github_env.mock=1 if args.mock else 0
   github_env.release_flag=1 if args.mock  or args.force or 'release' in commit_message.lower() else 0
   github_env.ui_build=1 if args.mock or args.ui_build or '[ui-build]' in commit_message.lower() or github_env.release_flag==1 else 0
@@ -502,13 +531,13 @@ def handle_build_flags(args):
 def handle_environment(args):
     set_workdir(args)
     logger.info('Setting environment variables...')
-
+    commit_message:str= Releases.get_last_commit_message()
     last:Commit = Releases.get_last_commit()
-    commit_message:str= last.message.replace('\n', ' ')
+    if last is not None:
-    github_env.author_name=last.author.name
+      github_env.author_name=last.author.name
-    github_env.author_email=last.author.email
+      github_env.author_email=last.author.email
-    github_env.committer_name=last.committer.name
+      github_env.committer_name=last.committer.name
-    github_env.committer_email=last.committer.email    
+      github_env.committer_email=last.committer.email    
     github_env.node=args.node
     github_env.depth=args.depth
     github_env.major=args.major
@@ -527,24 +556,23 @@ def handle_environment(args):
     github_env.artifact_file_name=f"{github_env.artifact_prefix}.zip"
     github_env.artifact_bin_file_name=f"{github_env.artifact_prefix}.bin"
     github_env.PROJECT_VER=f'{args.node}-{ args.build }'
-    commit_list = []
+    github_env.description=Releases.get_commit_list_descriptions()
-    for c in [c for i,c in enumerate(Releases.get_repository().walk(last.id,pygit2.GIT_SORT_TIME)) if i<10]:
-      commit_list.append(format_commit(c))
-    github_env.description='### Revision Log<br><<~EOD\n'+'<br>\n'.join(commit_list)+'\n~EOD'
     write_github_env(args)
 
 def handle_artifacts(args):
     set_workdir(args)
     logger.info(f'Handling artifacts')
     for attr in artifacts_formats:
-      target:str=attr[1].replace(artifacts_formats_outdir,args.outdir).replace(artifacts_formats_prefix,format_artifact_name())
+      target:str=os.path.relpath(attr[1].replace(artifacts_formats_outdir,args.outdir).replace(artifacts_formats_prefix,format_artifact_name()))
-      logger.debug(f'file {attr[0]} will be copied to {target}')
+      source:str=os.path.relpath(attr[0])
+      target_dir:str=os.path.dirname(target)
+      logger.info(f'Copying file {source} to {target}')
       try:
-        os.makedirs(os.path.dirname(target), exist_ok=True)
+        os.makedirs(target_dir, exist_ok=True)
-        shutil.copyfile(attr[0].rstrip(), target, follow_symlinks=True)
+        shutil.copyfile(source, target, follow_symlinks=True)
       except Exception as ex:
-        print(f'::error::Error while copying to {target}' )
+        print(f'::error::Error while copying {source} to {target}' )
-        print(f'::error::Content of {os.path.dirname(attr[0].rstrip())}:')
+        print(f'::error::Content of {target_dir}:')
         print('\n::error::'.join(get_file_list(os.path.dirname(attr[0].rstrip()))))
         raise