-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* update table format * improve table clarity * init code commit * doc: add flashattention installation guide and change toc * feat: remove libaio dependency * remove spdlog dependency * misc: remove unused code and dependencies * misc: remove commented-out code and unused imports * fix: cuda oom due to safe tensors open * remove gcc-12 requirement * gptq disable exllama * fix: key error in offload set * add forward and call (#7) * add forward and call * fix a bug * feat: support grok-1 model --------- Co-authored-by: xly <[email protected]> Co-authored-by: Zhan Lu <[email protected]> Co-authored-by: lausannel <[email protected]> Co-authored-by: Yao Fu <[email protected]>
- Loading branch information
1 parent
4509b87
commit 87eacf7
Showing
96 changed files
with
11,548 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,155 @@ | ||
--- | ||
# Refer to the following link for the explanation of each params: | ||
# http://releases.llvm.org/8.0.0/tools/clang/docs/ClangFormatStyleOptions.html | ||
Language: Cpp | ||
# BasedOnStyle: Google | ||
AccessModifierOffset: -4 | ||
AlignAfterOpenBracket: Align | ||
AlignConsecutiveAssignments: false | ||
AlignConsecutiveDeclarations: false | ||
AlignEscapedNewlines: Left | ||
AlignOperands: true | ||
AlignTrailingComments: true | ||
AllowAllParametersOfDeclarationOnNextLine: false | ||
AllowShortBlocksOnASingleLine: true | ||
AllowShortCaseLabelsOnASingleLine: true | ||
AllowShortFunctionsOnASingleLine: All | ||
AllowShortIfStatementsOnASingleLine: true | ||
AllowShortLoopsOnASingleLine: true | ||
# This is deprecated | ||
AlwaysBreakAfterDefinitionReturnType: None | ||
AlwaysBreakAfterReturnType: None | ||
AlwaysBreakBeforeMultilineStrings: true | ||
AlwaysBreakTemplateDeclarations: true | ||
BinPackArguments: false | ||
BinPackParameters: false | ||
BraceWrapping: | ||
AfterClass: false | ||
AfterControlStatement: false | ||
AfterEnum: false | ||
AfterFunction: false | ||
AfterNamespace: false | ||
AfterObjCDeclaration: false | ||
AfterStruct: false | ||
AfterUnion: false | ||
AfterExternBlock: false | ||
BeforeCatch: false | ||
BeforeElse: false | ||
IndentBraces: false | ||
# disabling the below splits, else, they'll just add to the vertical length of source files! | ||
SplitEmptyFunction: false | ||
SplitEmptyRecord: false | ||
SplitEmptyNamespace: false | ||
BreakBeforeBinaryOperators: None | ||
BreakBeforeBraces: WebKit | ||
BreakBeforeInheritanceComma: false | ||
BreakInheritanceList: BeforeColon | ||
BreakBeforeTernaryOperators: true | ||
BreakConstructorInitializersBeforeComma: false | ||
BreakConstructorInitializers: BeforeColon | ||
BreakAfterJavaFieldAnnotations: false | ||
BreakStringLiterals: true | ||
ColumnLimit: 100 | ||
CommentPragmas: '^ IWYU pragma:' | ||
CompactNamespaces: false | ||
ConstructorInitializerAllOnOneLineOrOnePerLine: true | ||
# Kept the below 2 to be the same as `IndentWidth` to keep everything uniform | ||
ConstructorInitializerIndentWidth: 4 | ||
ContinuationIndentWidth: 4 | ||
Cpp11BracedListStyle: true | ||
DerivePointerAlignment: false | ||
DisableFormat: false | ||
ExperimentalAutoDetectBinPacking: false | ||
FixNamespaceComments: true | ||
ForEachMacros: | ||
- foreach | ||
- Q_FOREACH | ||
- BOOST_FOREACH | ||
IncludeBlocks: Preserve | ||
IncludeCategories: | ||
- Regex: '^<ext/.*\.h>' | ||
Priority: 2 | ||
- Regex: '^<.*\.h>' | ||
Priority: 1 | ||
- Regex: '^<.*' | ||
Priority: 2 | ||
- Regex: '.*' | ||
Priority: 3 | ||
IncludeIsMainRegex: '([-_](test|unittest))?$' | ||
IndentCaseLabels: true | ||
IndentPPDirectives: None | ||
IndentWidth: 4 | ||
IndentWrappedFunctionNames: false | ||
JavaScriptQuotes: Leave | ||
JavaScriptWrapImports: true | ||
KeepEmptyLinesAtTheStartOfBlocks: false | ||
MacroBlockBegin: '' | ||
MacroBlockEnd: '' | ||
MaxEmptyLinesToKeep: 1 | ||
NamespaceIndentation: None | ||
ObjCBinPackProtocolList: Never | ||
ObjCBlockIndentWidth: 4 | ||
ObjCSpaceAfterProperty: false | ||
ObjCSpaceBeforeProtocolList: true | ||
PenaltyBreakAssignment: 4 | ||
PenaltyBreakBeforeFirstCallParameter: 1 | ||
PenaltyBreakComment: 300 | ||
PenaltyBreakFirstLessLess: 120 | ||
PenaltyBreakString: 1000 | ||
PenaltyBreakTemplateDeclaration: 10 | ||
PenaltyExcessCharacter: 1000000 | ||
PenaltyReturnTypeOnItsOwnLine: 200 | ||
PointerAlignment: Left | ||
RawStringFormats: | ||
- Language: Cpp | ||
Delimiters: | ||
- cc | ||
- CC | ||
- cpp | ||
- Cpp | ||
- CPP | ||
- 'c++' | ||
- 'C++' | ||
CanonicalDelimiter: '' | ||
- Language: TextProto | ||
Delimiters: | ||
- pb | ||
- PB | ||
- proto | ||
- PROTO | ||
EnclosingFunctions: | ||
- EqualsProto | ||
- EquivToProto | ||
- PARSE_PARTIAL_TEXT_PROTO | ||
- PARSE_TEST_PROTO | ||
- PARSE_TEXT_PROTO | ||
- ParseTextOrDie | ||
- ParseTextProtoOrDie | ||
CanonicalDelimiter: '' | ||
BasedOnStyle: google | ||
# Enabling comment reflow causes doxygen comments to be messed up in their formats! | ||
ReflowComments: true | ||
SortIncludes: true | ||
SortUsingDeclarations: true | ||
SpaceAfterCStyleCast: false | ||
SpaceAfterTemplateKeyword: true | ||
SpaceBeforeAssignmentOperators: true | ||
SpaceBeforeCpp11BracedList: false | ||
SpaceBeforeCtorInitializerColon: true | ||
SpaceBeforeInheritanceColon: true | ||
SpaceBeforeParens: ControlStatements | ||
SpaceBeforeRangeBasedForLoopColon: true | ||
SpaceInEmptyParentheses: false | ||
SpacesBeforeTrailingComments: 2 | ||
SpacesInAngles: false | ||
SpacesInContainerLiterals: true | ||
SpacesInCStyleCastParentheses: false | ||
SpacesInParentheses: false | ||
SpacesInSquareBrackets: false | ||
Standard: Cpp11 | ||
StatementMacros: | ||
- Q_UNUSED | ||
- QT_REQUIRE_VERSION | ||
# Be consistent with indent-width, even for people who use tab for indentation! | ||
TabWidth: 4 | ||
UseTab: Never |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
recursive-include core *.cpp *.h *.cc | ||
recursive-include op_builder *.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
// Copyright (c) TorchMoE. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
// TorchMoE Team | ||
|
||
#include "archer_aio_thread.h" | ||
|
||
#include "utils/archer_logger.h" | ||
|
||
ArcherAioThread::ArcherAioThread(int thread_id) : thread_id_(thread_id), is_running_(false) | ||
{ | ||
ARCHER_LOG_INFO("Create ArcherAioThread for thread: ", thread_id_); | ||
} | ||
|
||
ArcherAioThread::~ArcherAioThread() { Stop(); } | ||
|
||
void ArcherAioThread::Start() | ||
{ | ||
if (is_running_) { return; } | ||
|
||
is_running_ = true; | ||
pending_callbacks_ = 0; | ||
thread_ = std::thread(&ArcherAioThread::Run, this); | ||
} | ||
|
||
void ArcherAioThread::Stop() | ||
{ | ||
if (!is_running_) { return; } | ||
|
||
is_running_ = false; | ||
thread_.join(); | ||
} | ||
|
||
void ArcherAioThread::Enqueue(AioCallback& callback) | ||
{ | ||
std::lock_guard<std::mutex> lock(mutex_); | ||
callbacks_.push_back(std::move(callback)); | ||
pending_callbacks_.fetch_add(1); | ||
} | ||
|
||
void ArcherAioThread::Wait() | ||
{ | ||
// while (!callbacks_.empty()) { usleep(1000); } | ||
while (pending_callbacks_.load() != 0) { usleep(1000); } | ||
std::lock_guard<std::mutex> lock(mutex_); | ||
callbacks_.clear(); | ||
} | ||
|
||
void ArcherAioThread::Run() | ||
{ | ||
|
||
while (is_running_) { | ||
std::function<void()> callback; | ||
{ | ||
std::lock_guard<std::mutex> lock(mutex_); | ||
if (callbacks_.empty()) { continue; } | ||
callback = std::move(callbacks_.front()); | ||
callbacks_.pop_front(); | ||
} | ||
callback(); | ||
pending_callbacks_.fetch_sub(1); | ||
} | ||
|
||
} |
Oops, something went wrong.